/** * utility function to extract timezone id from Druid query * @param query Druid Rel * @return time zone */ private static TimeZone timezoneId(final DruidQuery query, final RexNode arg) { return arg.getType().getSqlTypeName() == SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE ? TimeZone.getTimeZone( query.getTopNode().getCluster().getPlanner().getContext().unwrap(CalciteConnectionConfig.class).timeZone()) : TimeZone.getTimeZone("UTC"); }
@Override public final HiveJoin copy(RelTraitSet traitSet, RexNode conditionExpr, RelNode left, RelNode right, JoinRelType joinType, boolean semiJoinDone) { try { Set<String> variablesStopped = Collections.emptySet(); HiveJoin join = new HiveJoin(getCluster(), traitSet, left, right, conditionExpr, joinType, variablesStopped, joinAlgorithm); // If available, copy state to registry for optimization rules HiveRulesRegistry registry = join.getCluster().getPlanner().getContext().unwrap(HiveRulesRegistry.class); if (registry != null) { registry.copyPushedPredicates(this, join); } return join; } catch (InvalidRelException | CalciteSemanticException e) { // Semantic error not possible. Must be a bug. Convert to // internal error. throw new AssertionError(e); } }
@Override public final HiveJoin copy(RelTraitSet traitSet, RexNode conditionExpr, RelNode left, RelNode right, JoinRelType joinType, boolean semiJoinDone) { try { Set<String> variablesStopped = Collections.emptySet(); HiveJoin join = new HiveJoin(getCluster(), traitSet, left, right, conditionExpr, joinType, variablesStopped, joinAlgorithm); // If available, copy state to registry for optimization rules HiveRulesRegistry registry = join.getCluster().getPlanner().getContext().unwrap(HiveRulesRegistry.class); if (registry != null) { registry.copyPushedPredicates(this, join); } return join; } catch (InvalidRelException | CalciteSemanticException e) { // Semantic error not possible. Must be a bug. Convert to // internal error. throw new AssertionError(e); } }
public boolean initialize(RexNode condition, DrillScanRelBase scanRel, IndexCallContext context) { GroupScan scan = IndexPlanUtils.getGroupScan(scanRel); PlannerSettings settings = PrelUtil.getPlannerSettings(scanRel.getCluster().getPlanner()); rowKeyJoinBackIOFactor = settings.getIndexRowKeyJoinCostFactor(); if (scan instanceof DbGroupScan) { String conditionAsStr = convertRexToString(condition, scanRel.getRowType()); if (statsCache.get(conditionAsStr) == null) { IndexCollection indexes = ((DbGroupScan)scan).getSecondaryIndexCollection(scanRel); populateStats(condition, indexes, scanRel, context); logger.info("index_plan_info: initialize: scanRel #{} and groupScan {} got fulltable {}, statsCache: {}, fiStatsCache: {}", scanRel.getId(), System.identityHashCode(scan), fullTableScanPayload, statsCache, fIStatsCache); return true; } } return false; }
public static Integer getSplitCountWithRepartition(HiveJoin join) { final Double maxSplitSize = join.getCluster().getPlanner().getContext(). unwrap(HiveAlgorithmsConf.class).getMaxSplitSize(); // We repartition: new number of splits RelMetadataQuery mq = RelMetadataQuery.instance(); final Double averageRowSize = mq.getAverageRowSize(join); final Double rowCount = mq.getRowCount(join); if (averageRowSize == null || rowCount == null) { return null; } final Double totalSize = averageRowSize * rowCount; final Double splitCount = totalSize / maxSplitSize; return splitCount.intValue(); }
@Override public boolean isExecutable(HiveJoin join) { final Double maxMemory = join.getCluster().getPlanner().getContext(). unwrap(HiveAlgorithmsConf.class).getMaxMemory(); // Check streaming side RelNode smallInput = join.getStreamingInput(); if (smallInput == null) { return false; } return HiveAlgorithmsUtil.isFittingIntoMemory(maxMemory, smallInput, 1); }
public static Integer getSplitCountWithRepartition(HiveJoin join) { final Double maxSplitSize = join.getCluster().getPlanner().getContext(). unwrap(HiveAlgorithmsConf.class).getMaxSplitSize(); // We repartition: new number of splits final RelMetadataQuery mq = join.getCluster().getMetadataQuery(); final Double averageRowSize = mq.getAverageRowSize(join); final Double rowCount = mq.getRowCount(join); if (averageRowSize == null || rowCount == null) { return null; } final Double totalSize = averageRowSize * rowCount; final Double splitCount = totalSize / maxSplitSize; return splitCount.intValue(); }
@Override public SemiJoin copy(RelTraitSet traitSet, RexNode condition, RelNode left, RelNode right, JoinRelType joinType, boolean semiJoinDone) { try { final JoinInfo joinInfo = JoinInfo.of(left, right, condition); HiveSemiJoin semijoin = new HiveSemiJoin(getCluster(), traitSet, left, right, condition, joinInfo.leftKeys, joinInfo.rightKeys); // If available, copy state to registry for optimization rules HiveRulesRegistry registry = semijoin.getCluster().getPlanner().getContext().unwrap(HiveRulesRegistry.class); if (registry != null) { registry.copyPushedPredicates(this, semijoin); } return semijoin; } catch (InvalidRelException | CalciteSemanticException e) { // Semantic error not possible. Must be a bug. Convert to // internal error. throw new AssertionError(e); } }
@Override public boolean isExecutable(HiveJoin join) { final Double maxMemory = join.getCluster().getPlanner().getContext(). unwrap(HiveAlgorithmsConf.class).getMaxMemory(); // Check streaming side RelNode smallInput = join.getStreamingInput(); if (smallInput == null) { return false; } return HiveAlgorithmsUtil.isFittingIntoMemory(maxMemory, smallInput, 1); }
@Override public SemiJoin copy(RelTraitSet traitSet, RexNode condition, RelNode left, RelNode right, JoinRelType joinType, boolean semiJoinDone) { try { final JoinInfo joinInfo = JoinInfo.of(left, right, condition); HiveSemiJoin semijoin = new HiveSemiJoin(getCluster(), traitSet, left, right, condition, joinInfo.leftKeys, joinInfo.rightKeys); // If available, copy state to registry for optimization rules HiveRulesRegistry registry = semijoin.getCluster().getPlanner().getContext().unwrap(HiveRulesRegistry.class); if (registry != null) { registry.copyPushedPredicates(this, semijoin); } return semijoin; } catch (InvalidRelException | CalciteSemanticException e) { // Semantic error not possible. Must be a bug. Convert to // internal error. throw new AssertionError(e); } }
/** Decorrelates a query. * * <p>This is the main entry point to {@code RelDecorrelator}. * * @param rootRel Root node of the query * * @return Equivalent query with all * {@link org.apache.calcite.rel.logical.LogicalCorrelate} instances removed */ public static RelNode decorrelateQuery(RelNode rootRel) { final CorelMap corelMap = new CorelMapBuilder().build(rootRel); if (!corelMap.hasCorrelation()) { return rootRel; } final RelOptCluster cluster = rootRel.getCluster(); final HiveRelDecorrelator decorrelator = new HiveRelDecorrelator(cluster, corelMap, cluster.getPlanner().getContext()); RelNode newRootRel = decorrelator.removeCorrelationViaRule(rootRel); if (!decorrelator.cm.mapCorToCorRel.isEmpty()) { newRootRel = decorrelator.decorrelate(newRootRel); } return newRootRel; }
@Override public boolean matches(RelOptRuleCall call) { final DrillScanRel scan = (DrillScanRel) call.rel(2); GroupScan groupScan = scan.getGroupScan(); // this rule is applicable only for Hive based partition pruning if (PrelUtil.getPlannerSettings(scan.getCluster().getPlanner()).isHepPartitionPruningEnabled()) { return groupScan instanceof HiveScan && groupScan.supportsPartitionFilterPushdown() && !scan.partitionFilterPushdown(); } else { return groupScan instanceof HiveScan && groupScan.supportsPartitionFilterPushdown(); } }
/** Decorrelates a query. * * <p>This is the main entry point to {@code RelDecorrelator}. * * @param rootRel Root node of the query * * @return Equivalent query with all * {@link org.apache.calcite.rel.logical.LogicalCorrelate} instances removed */ public static RelNode decorrelateQuery(RelNode rootRel) { final CorelMap corelMap = new CorelMapBuilder().build(rootRel); if (!corelMap.hasCorrelation()) { return rootRel; } final RelOptCluster cluster = rootRel.getCluster(); final HiveRelDecorrelator decorrelator = new HiveRelDecorrelator(cluster, corelMap, cluster.getPlanner().getContext()); RelNode newRootRel = decorrelator.removeCorrelationViaRule(rootRel); if (!decorrelator.cm.mapCorToCorRel.isEmpty()) { newRootRel = decorrelator.decorrelate(newRootRel); } return newRootRel; }
@Override public boolean matches(RelOptRuleCall call) { final DrillScanRel scan = (DrillScanRel) call.rel(1); GroupScan groupScan = scan.getGroupScan(); // this rule is applicable only for Hive based partition pruning if (PrelUtil.getPlannerSettings(scan.getCluster().getPlanner()).isHepPartitionPruningEnabled()) { return groupScan instanceof HiveScan && groupScan.supportsPartitionFilterPushdown() && !scan.partitionFilterPushdown(); } else { return groupScan instanceof HiveScan && groupScan.supportsPartitionFilterPushdown(); } }
basePlan.getCluster().getPlanner().getContext()); basePlan.getCluster().getPlanner().setExecutor(executorProvider); planner.setExecutor(executorProvider);
/** Returns if one rel is weaker than another. */ protected boolean isWeaker(MutableRel rel0, MutableRel rel) { if (rel0 == rel || equivalents.get(rel0).contains(rel)) { return false; } if (!(rel0 instanceof MutableFilter) || !(rel instanceof MutableFilter)) { return false; } if (!rel.getRowType().equals(rel0.getRowType())) { return false; } final MutableRel rel0input = ((MutableFilter) rel0).getInput(); final MutableRel relinput = ((MutableFilter) rel).getInput(); if (rel0input != relinput && !equivalents.get(rel0input).contains(relinput)) { return false; } RexExecutorImpl rexImpl = (RexExecutorImpl) (rel.cluster.getPlanner().getExecutor()); RexImplicationChecker rexImplicationChecker = new RexImplicationChecker( rel.cluster.getRexBuilder(), rexImpl, rel.getRowType()); return rexImplicationChecker.implies(((MutableFilter) rel0).getCondition(), ((MutableFilter) rel).getCondition()); }
@Override public boolean isExecutable(HiveJoin join) { final RelMetadataQuery mq = join.getCluster().getMetadataQuery(); final Double maxMemory = join.getCluster().getPlanner().getContext(). unwrap(HiveAlgorithmsConf.class).getMaxMemory();
listBuilder.add(newCondition); RexExecutor executor = Util.first(filterRel.getCluster().getPlanner().getExecutor(), RexUtil.EXECUTOR); final RexSimplify simplify = new RexSimplify(rexBuilder, true, executor);
SubqueryConf subqueryConfig = filter.getCluster().getPlanner(). getContext().unwrap(SubqueryConf.class); boolean isCorrScalarQuery = subqueryConfig.getCorrScalarRexSQWithAgg().contains(e.rel); project.getCluster().getPlanner().getContext().unwrap(SubqueryConf.class); boolean isCorrScalarQuery = subqueryConfig.getCorrScalarRexSQWithAgg().contains(e.rel); boolean hasNoWindowingAndNoGby =
public void onMatch(RelOptRuleCall call) { final Filter filter = call.rel(0); //final RelBuilder builder = call.builder(); //TODO: replace HiveSubQRemoveRelBuilder with calcite's once calcite 1.11.0 is released final HiveSubQRemoveRelBuilder builder = new HiveSubQRemoveRelBuilder(null, call.rel(0).getCluster(), null); final RexSubQuery e = RexUtil.SubQueryFinder.find(filter.getCondition()); assert e != null; final RelOptUtil.Logic logic = LogicVisitor.find(RelOptUtil.Logic.TRUE, ImmutableList.of(filter.getCondition()), e); builder.push(filter.getInput()); final int fieldCount = builder.peek().getRowType().getFieldCount(); assert(filter instanceof HiveFilter); Set<RelNode> corrScalarQueries = filter.getCluster().getPlanner().getContext().unwrap(Set.class); boolean isCorrScalarQuery = corrScalarQueries.contains(e.rel); final RexNode target = apply(e, HiveFilter.getVariablesSet(e), logic, builder, 1, fieldCount, isCorrScalarQuery); final RexShuttle shuttle = new ReplaceSubQueryShuttle(e, target); builder.filter(shuttle.apply(filter.getCondition())); builder.project(fields(builder, filter.getRowType().getFieldCount())); call.transformTo(builder.build()); } };