HivePlannerContext confContext = new HivePlannerContext(algorithmsConf, registry, corrScalarRexSQWithAgg); RelOptPlanner planner = HiveVolcanoPlanner.createPlanner(confContext); final RexBuilder rexBuilder = cluster.getRexBuilder(); final RelOptCluster optCluster = RelOptCluster.create(planner, rexBuilder); LOG.debug("Plan after decorrelation:\n" + RelOptUtil.toString(calciteGenPlan)); list.add(mdProvider.getMetadataProvider()); RelTraitSet desiredTraits = optCluster .traitSetOf(HiveRelNode.CONVENTION, RelCollations.EMPTY); if (!calcitePreCboPlan.getTraitSet().equals(desiredTraits)) { rootRel = hepPlanner.changeTraits(calcitePreCboPlan, desiredTraits); planner.registerMetadataProviders(Lists.newArrayList(calciteMdProvider)); planner.addMaterialization(materialization); planner.addRule(HiveMaterializedViewFilterScanRule.INSTANCE); planner.setRoot(calciteOptimizedPlan); calciteOptimizedPlan = planner.findBestExp(); planner.clear();
public void onMatch(RelOptRuleCall call) { final Filter filter = call.rel(0); //final RelBuilder builder = call.builder(); //TODO: replace HiveSubQRemoveRelBuilder with calcite's once calcite 1.11.0 is released final HiveSubQRemoveRelBuilder builder = new HiveSubQRemoveRelBuilder(null, call.rel(0).getCluster(), null); final RexSubQuery e = RexUtil.SubQueryFinder.find(filter.getCondition()); assert e != null; final RelOptUtil.Logic logic = LogicVisitor.find(RelOptUtil.Logic.TRUE, ImmutableList.of(filter.getCondition()), e); builder.push(filter.getInput()); final int fieldCount = builder.peek().getRowType().getFieldCount(); assert(filter instanceof HiveFilter); Set<RelNode> corrScalarQueries = filter.getCluster().getPlanner().getContext().unwrap(Set.class); boolean isCorrScalarQuery = corrScalarQueries.contains(e.rel); final RexNode target = apply(e, HiveFilter.getVariablesSet(e), logic, builder, 1, fieldCount, isCorrScalarQuery); final RexShuttle shuttle = new ReplaceSubQueryShuttle(e, target); builder.filter(shuttle.apply(filter.getCondition())); builder.project(fields(builder, filter.getRowType().getFieldCount())); call.transformTo(builder.build()); } };
@Override public RelOptCost computeSelfCost(final RelOptPlanner planner, final RelMetadataQuery mq) { return planner.getCostFactory().makeCost(rels.stream().mapToDouble(mq::getRowCount).sum(), 0, 0); }
planner.addRule(OLAPToEnumerableConverterRule.INSTANCE); planner.addRule(OLAPFilterRule.INSTANCE); planner.addRule(OLAPProjectRule.INSTANCE); planner.addRule(OLAPAggregateRule.INSTANCE); planner.addRule(OLAPJoinRule.INSTANCE); planner.addRule(OLAPLimitRule.INSTANCE); planner.addRule(OLAPSortRule.INSTANCE); planner.addRule(OLAPUnionRule.INSTANCE); planner.addRule(OLAPWindowRule.INSTANCE); planner.addRule(OLAPValuesRule.INSTANCE); planner.addRule(AggregateProjectReduceRule.INSTANCE); planner.addRule(ReduceExpressionsRule.PROJECT_INSTANCE); planner.addRule(ReduceExpressionsRule.FILTER_INSTANCE); planner.addRule(ReduceExpressionsRule.CALC_INSTANCE); planner.addRule(ReduceExpressionsRule.JOIN_INSTANCE); if (!kylinConfig.isEnumerableRulesEnabled()) { for (RelOptRule rule : CalcitePrepareImpl.ENUMERABLE_RULES) { planner.removeRule(rule); planner.removeRule(FilterJoinRule.FILTER_ON_JOIN); planner.removeRule(FilterJoinRule.JOIN); planner.removeRule(JoinCommuteRule.INSTANCE); planner.removeRule(JoinPushThroughJoinRule.LEFT); planner.removeRule(JoinPushThroughJoinRule.RIGHT);
private RelNode applyMaterializedViewRewriting(RelOptPlanner planner, RelNode basePlan, RelMetadataProvider mdProvider, RexExecutor executorProvider) { final RelOptCluster optCluster = basePlan.getCluster(); final PerfLogger perfLogger = SessionState.getPerfLogger(); optCluster.invalidateMetadataQuery(); RelMetadataQuery.THREAD_PROVIDERS.set(JaninoRelMetadataProvider.of(DefaultRelMetadataProvider.INSTANCE)); planner.addMaterialization(materialization); planner.addRule(HiveMaterializedViewRule.INSTANCE_PROJECT_FILTER); planner.addRule(HiveMaterializedViewRule.INSTANCE_FILTER); planner.addRule(HiveMaterializedViewRule.INSTANCE_PROJECT_JOIN); planner.addRule(HiveMaterializedViewRule.INSTANCE_JOIN); planner.addRule(HiveMaterializedViewRule.INSTANCE_PROJECT_AGGREGATE); planner.addRule(HiveMaterializedViewRule.INSTANCE_AGGREGATE); planner.addRule(HiveFilterProjectTSTransposeRule.INSTANCE); planner.addRule(new HivePartitionPruneRule(conf)); planner.setRoot(basePlan); basePlan = planner.findBestExp(); planner.clear(); optCluster.invalidateMetadataQuery(); RelMetadataQuery.THREAD_PROVIDERS.set(JaninoRelMetadataProvider.of(mdProvider)); if (!RelOptUtil.toString(calcitePreMVRewritingPlan).equals(RelOptUtil.toString(basePlan))) {
if (followPlanChanges) { programBuilder.addMatchOrder(order); programBuilder = programBuilder.addRuleCollection(ImmutableList.copyOf(rules)); } else { basePlan.getCluster().getPlanner().getContext()); planner.registerMetadataProviders(list); RelMetadataProvider chainedProvider = ChainedRelMetadataProvider.of(list); basePlan.getCluster().setMetadataProvider( new CachingRelMetadataProvider(chainedProvider, planner)); basePlan.getCluster().getPlanner().setExecutor(executorProvider); planner.setExecutor(executorProvider);
protected void apply(RelOptRuleCall call, Project postProject, final Aggregate aggregate, StarTable.StarTableScan scan) { final RelOptPlanner planner = call.getPlanner(); final CalciteConnectionConfig config = planner.getContext().unwrap(CalciteConnectionConfig.class); if (config == null || !config.createMaterializations()) { final RelOptLattice lattice = planner.getLattice(table); final List<Lattice.Measure> measures = lattice.lattice.toMeasures(aggregate.getAggCallList()); return; tableEntry, rowCount); relBuilder.push(aggregateRelOptTable.toRel(RelOptUtil.getContext(cluster))); if (tileKey == null) { if (CalcitePrepareImpl.DEBUG) { aggregate.copy(aggregate.getTraitSet(), relBuilder.build(), false, groupSet.build(), null, aggCalls)); } else if (!tileKey.measures.equals(measures)) { if (CalcitePrepareImpl.DEBUG) { System.out.println("Using materialization " relBuilder.push( postProject.copy(postProject.getTraitSet(), ImmutableList.of(relBuilder.peek())));
public RelNode apply(RelOptCluster cluster, RelOptSchema relOptSchema, SchemaPlus rootSchema) { final RelDataTypeFactory typeFactory = cluster.getTypeFactory(); final RexBuilder rexBuilder = cluster.getRexBuilder(); final RelOptPlanner planner = cluster.getPlanner(); ImmutableList.of( (RexNode) rexBuilder.makeInputRef(stringType, 0), rexBuilder.makeInputRef(integerType, 1)), RelOptUtil.dumpPlan("LOGICAL PLAN", rootRel, SqlExplainFormat.TEXT, SqlExplainLevel.DIGEST_ATTRIBUTES); RelTraitSet desiredTraits = rootRel.getTraitSet().replace(PHYSICAL); final RelNode rootRel2 = planner.changeTraits(rootRel, desiredTraits); planner.setRoot(rootRel2); return planner.findBestExp();
public void onMatch(RelOptRuleCall call) { final LogicalCorrelate correlate = call.rel(0); final RelNode left = call.rel(1); final Aggregate aggregate = call.rel(2); final Project project = call.rel(3); RelNode right = call.rel(4); final RelOptCluster cluster = correlate.getCluster(); setCurrent(call.getPlanner().getRoot(), correlate); if (RelOptUtil.getVariablesUsed(right).size() > 0) { return; left.getRowType().getFieldCount() + rightJoinKeys.get(0).getIndex(); } else if (cm.mapRefRelToCorRef.containsKey(project)) { if (RelOptUtil.getVariablesUsed(right).size() > 0) { return; ImmutableList.of( Pair.<RexNode, String>of( rexBuilder.makeLiteral(true), "nullIndicator"))); left.getRowType().getFieldCount() + right.getRowType().getFieldCount() - 1; } else { return;
private static RelNode toPhysicalPlan(RelNode root, RuleSet rules) { Program program = Programs.of(rules); RelOptPlanner plan = root.getCluster().getPlanner(); RelTraitSet traits = plan.emptyTraitSet().replace(EnumerableConvention.INSTANCE); return program.run(plan, root, traits, ImmutableList.<RelOptMaterialization>of(), ImmutableList.<RelOptLattice>of()); }
@Override public void onMatch(RelOptRuleCall call) { final Join join = call.rel(0); final HepRelVertex root = (HepRelVertex) call.getPlanner().getRoot(); if (root.getCurrentRel() != join) { // Bail out return; } // The join is the root, but we should always end up with a Project operator // on top. We will add it. RelBuilder relBuilder = call.builder(); relBuilder.push(join); List<RexNode> identityFields = relBuilder.fields( ImmutableBitSet.range(0, join.getRowType().getFieldCount()).asList()); relBuilder.project(identityFields, ImmutableList.<String>of(), true); call.transformTo(relBuilder.build()); }
final RelOptCluster cluster = correlate.getCluster(); setCurrent(call.getPlanner().getRoot(), correlate); if (RelOptUtil.getVariablesUsed(right).size() > 0) { return; RelOptUtil.splitCorrelatedFilterCondition( filter, rightJoinKeys, } else if (cm.mapRefRelToCorRef.containsKey(aggInputProject)) { if (RelOptUtil.getVariablesUsed(right).size() > 0) { return; int nFields = left.getRowType().getFieldCount(); ImmutableBitSet allCols = ImmutableBitSet.range(nFields); RelDataType leftInputFieldType = left.getRowType(); int leftInputFieldCount = leftInputFieldType.getFieldCount(); int joinOutputProjExprCount = ImmutableList.of( Pair.of(rexBuilder.makeLiteral(true), "nullIndicator")));
@Override public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { double dRows = mq.getRowCount(getInput()); double dCpu = dRows * exps.size(); double dIo = 0; return planner.getCostFactory().makeCost(dRows, dCpu, dIo); }
@Override public void onMatch(RelOptRuleCall call) { Join join = call.rel(0); RelOptPredicateList preds = call.getMetadataQuery().getPulledUpPredicates(join); HiveRulesRegistry registry = call.getPlanner().getContext().unwrap(HiveRulesRegistry.class); assert registry != null; RexBuilder rB = join.getCluster().getRexBuilder(); RelNode lChild = join.getLeft(); RelNode rChild = join.getRight(); leftPushedPredicates, preds.leftInferredPredicates, lChild.getRowType()); Set<String> rightPushedPredicates = Sets.newHashSet(registry.getPushedPredicates(join, 1)); List<RexNode> rightPreds = getValidPreds(join.getCluster(), rChild, rightPushedPredicates, preds.rightInferredPredicates, rChild.getRowType()); lChild = filterFactory.createFilter( lChild, newLeftPredicate.accept(new RexReplacer(lChild))); call.getPlanner().onCopy(curr, lChild); rChild = filterFactory.createFilter( rChild, newRightPredicate.accept(new RexReplacer(rChild))); call.getPlanner().onCopy(curr, rChild); call.getPlanner().onCopy(join, newRel);
final RexBuilder rexBuilder = cluster.getRexBuilder(); final RelOptCluster optCluster = RelOptCluster.create(planner, rexBuilder); calciteGenPlan.getCluster().getPlanner().setExecutor(executorProvider); LOG.debug("Plan before removing subquery:\n" + RelOptUtil.toString(calciteGenPlan)); calciteGenPlan = hepPlan(calciteGenPlan, false, mdProvider.getMetadataProvider(), null, new HiveSubQueryRemoveRule(conf)); LOG.debug("Plan just after removing subquery:\n" + RelOptUtil.toString(calciteGenPlan)); LOG.debug("Plan after decorrelation:\n" + RelOptUtil.toString(calciteGenPlan)); if (!calcitePreCboPlan.getTraitSet().equals(desiredTraits)) { rootRel = hepPlanner.changeTraits(calcitePreCboPlan, desiredTraits);
if (debug) { System.out.println( RelOptUtil.dumpPlan("-- Logical Plan", root, SqlExplainFormat.TEXT, SqlExplainLevel.DIGEST_ATTRIBUTES)); RelOptCluster cluster = root.getCluster(); final RelOptPlanner optPlanner = cluster.getPlanner(); cluster.traitSet().replace(EnumerableConvention.INSTANCE); final RelNode newRoot = optPlanner.changeTraits(root, desiredTraits); if (debug) { System.out.println( RelOptUtil.dumpPlan("-- Mid Plan", newRoot, SqlExplainFormat.TEXT, SqlExplainLevel.DIGEST_ATTRIBUTES)); optPlanner.setRoot(newRoot); RelNode bestExp = optPlanner.findBestExp(); if (debug) { System.out.println( RelOptUtil.dumpPlan("-- Best Plan", bestExp, SqlExplainFormat.TEXT, SqlExplainLevel.DIGEST_ATTRIBUTES));
public void onMatch(RelOptRuleCall call) { final LogicalCorrelate correlate = call.rel(0); final RelNode left = call.rel(1); final LogicalProject aggOutputProject = call.rel(2); final RelOptCluster cluster = correlate.getCluster(); setCurrent(call.getPlanner().getRoot(), correlate); if (RelOptUtil.getVariablesUsed(right).size() > 0) { return; RelOptUtil.splitCorrelatedFilterCondition( filter, rightJoinKeys, } else if (cm.mapRefRelToCorRef.containsKey(aggInputProject)) { if (RelOptUtil.getVariablesUsed(right).size() > 0) { return; int nFields = left.getRowType().getFieldCount(); ImmutableBitSet allCols = ImmutableBitSet.range(nFields); RelDataType leftInputFieldType = left.getRowType(); int leftInputFieldCount = leftInputFieldType.getFieldCount(); int joinOutputProjExprCount =
public RelNode run(RelOptPlanner planner, RelNode rel, RelTraitSet requiredOutputTraits, List<RelOptMaterialization> materializations, List<RelOptLattice> lattices) { planner.clear(); for (RelOptRule rule : ruleSet) { planner.addRule(rule); } for (RelOptMaterialization materialization : materializations) { planner.addMaterialization(materialization); } for (RelOptLattice lattice : lattices) { planner.addLattice(lattice); } if (!rel.getTraitSet().equals(requiredOutputTraits)) { rel = planner.changeTraits(rel, requiredOutputTraits); } planner.setRoot(rel); return planner.findBestExp(); } }
/** * Converts one trait of a relational expression, if it does not * already have that trait. * * @param rel Relational expression to convert * @param toTrait Desired trait * @return a relational expression with the desired trait; never null */ public static RelNode convert(RelNode rel, RelTrait toTrait) { RelOptPlanner planner = rel.getCluster().getPlanner(); RelTraitSet outTraits = rel.getTraitSet(); if (toTrait != null) { outTraits = outTraits.replace(toTrait); } if (rel.getTraitSet().matches(outTraits)) { return rel; } return planner.changeTraits(rel, outTraits.simplify()); }
@Override public void onMatch(RelOptRuleCall call) { final RelNode node = call.rel(0); numberOnMatch++; // If we have fired it already once, we return and the test will fail if (numberOnMatch > 1) { return; } // Register that we have visited this operator in this rule HiveRulesRegistry registry = call.getPlanner(). getContext().unwrap(HiveRulesRegistry.class); if (registry != null) { registry.registerVisited(this, node); } // We create a new op if it is the first time we fire the rule final RelNode newNode = new DummyNode(node.getCluster(), node.getTraitSet()); // We register it so we do not fire the rule on it again if (registry != null) { registry.registerVisited(this, newNode); } call.transformTo(newNode); } }