/** * This generates a logical plan. * * @param expr A relational algebraic expression for a query. * @return A logical plan */ public LogicalPlan createPlan(OverridableConf context, Expr expr) throws TajoException { return createPlan(context, expr, false); }
public void start() { try { analyzer = new SQLAnalyzer(); preVerifier = new PreLogicalPlanVerifier(context.getCatalog()); planner = new LogicalPlanner(context.getCatalog(), TablespaceManager.getInstance()); // Access path rewriter is enabled only in QueryMasterTask optimizer = new LogicalOptimizer(context.getConf(), context.getCatalog(), TablespaceManager.getInstance()); annotatedPlanVerifier = new LogicalPlanVerifier(); postLogicalPlanVerifier = new PostLogicalPlanVerifier(); } catch (Throwable t) { LOG.error(t.getMessage(), t); throw new RuntimeException(t); } super.start(); }
public static boolean isEvaluatableJoinQual(QueryBlock block, EvalNode evalNode, JoinNode node, boolean isOnPredicate, boolean isTopMostJoin) { if (checkIfBeEvaluatedAtJoin(block, evalNode, node, isTopMostJoin)) { if (isNonEquiThetaJoinQual(block, node, evalNode)) { return false; } if (PlannerUtil.isOuterJoinType(node.getJoinType())) { /* * For outer joins, only predicates which are specified at the on clause can be evaluated during processing join. * Other predicates from the where clause must be evaluated after the join. * The below code will be modified after improving join operators to keep join filters by themselves (TAJO-1310). */ if (!isOnPredicate) { return false; } } else { /* * Only join predicates should be evaluated at join if the join type is inner or cross. (TAJO-1445) */ if (!EvalTreeUtil.isJoinQual(block, node.getLeftChild().getOutSchema(), node.getRightChild().getOutSchema(), evalNode, false)) { return false; } } return true; } return false; }
private Pair<String [], ExprNormalizer.WindowSpecReferences []> doProjectionPrephase(PlanContext context, Projection projection) throws TajoException { QueryBlock block = context.queryBlock; int finalTargetNum = projection.size(); String [] referenceNames = new String[finalTargetNum]; ExprNormalizedResult[] normalizedExprList = new ExprNormalizedResult[finalTargetNum]; List<ExprNormalizer.WindowSpecReferences> windowSpecReferencesList = new ArrayList<>(); List<Integer> targetsIds = normalize(context, projection, normalizedExprList, new Matcher() { @Override public boolean isMatch(Expr expr) { return ExprFinder.finds(expr, OpType.WindowFunction).size() == 0; } }); // Note: Why separate normalization and add(Named)Expr? // // ExprNormalizer internally makes use of the named exprs in NamedExprsManager. // If we don't separate normalization work and addExprWithName, addExprWithName will find named exprs evaluated // the same logical node. It will cause impossible evaluation in physical executors. addNamedExprs(block, referenceNames, normalizedExprList, windowSpecReferencesList, projection, targetsIds); targetsIds = normalize(context, projection, normalizedExprList, new Matcher() { @Override public boolean isMatch(Expr expr) { return ExprFinder.finds(expr, OpType.WindowFunction).size() > 0; } }); addNamedExprs(block, referenceNames, normalizedExprList, windowSpecReferencesList, projection, targetsIds); return new Pair<>(referenceNames, windowSpecReferencesList.toArray(new ExprNormalizer.WindowSpecReferences[windowSpecReferencesList.size()])); }
public LogicalNode visitInsert(PlanContext context, Stack<Expr> stack, Insert expr) throws TajoException { stack.push(expr); LogicalNode subQuery = super.visitInsert(context, stack, expr); stack.pop(); InsertNode insertNode = context.queryBlock.getNodeFromExpr(expr); insertNode.setOverwrite(expr.isOverwrite()); insertNode.setSubQuery(subQuery); if (expr.hasTableName()) { // INSERT (OVERWRITE) INTO TABLE ... return buildInsertIntoTablePlan(context, insertNode, expr); } else if (expr.hasLocation()) { // INSERT (OVERWRITE) INTO LOCATION ... return buildInsertIntoLocationPlan(context, insertNode, expr); } else { throw new IllegalStateException("Invalid Query"); } }
private void setTargetOfTableSubQuery (PlanContext context, QueryBlock block, TableSubQueryNode subQueryNode) throws TajoException { // Add additional expressions required in upper nodes. Set<String> newlyEvaluatedExprs = new HashSet<>(); for (NamedExpr rawTarget : block.namedExprsMgr.getAllNamedExprs()) { try { EvalNode evalNode = exprAnnotator.createEvalNode(context, rawTarget.getExpr(), NameResolvingMode.RELS_ONLY); if (checkIfBeEvaluatedAtRelation(block, evalNode, subQueryNode)) { block.namedExprsMgr.markAsEvaluated(rawTarget.getAlias(), evalNode); newlyEvaluatedExprs.add(rawTarget.getAlias()); // newly added exr } } catch (UndefinedColumnException ve) { } } // Assume that each unique expr is evaluated once. LinkedHashSet<Target> targets = createFieldTargetsFromRelation(block, subQueryNode, newlyEvaluatedExprs); for (String newAddedExpr : newlyEvaluatedExprs) { targets.add(block.namedExprsMgr.getTarget(newAddedExpr, true)); } subQueryNode.setTargets(new ArrayList<>(targets)); }
LogicalPlanner planner = new LogicalPlanner(catalog, TablespaceManager.getInstance()); LogicalOptimizer optimizer = new LogicalOptimizer(systemConf, catalog, TablespaceManager.getInstance()); Expr expr = JsonHelper.fromJson(jsonExpr, Expr.class); plan = planner.createPlan(queryContext, expr); optimizer.optimize(queryContext, plan);
LogicalPlan newPlan = planner.createPlan(defaultContext, expr); LogicalNode plan = newPlan.getRootBlock().getRoot(); assertTrue(LogicalPlanner.checkIfBeEvaluatedAtJoin(newPlan.getRootBlock(), selNode.getQual(), joinNode, false)); newPlan = planner.createPlan(defaultContext, expr); plan = newPlan.getRootBlock().getRoot(); ScanNode scanNode = selNode.getChild(); assertTrue(LogicalPlanner.checkIfBeEvaluatedAtRelation(newPlan.getRootBlock(), selNode.getQual(), scanNode));
public LogicalNode visitScan(Context context, LogicalPlan plan, LogicalPlan.QueryBlock block, ScanNode node, Stack<LogicalNode> stack) throws TajoException { Context newContext = new Context(context); Target [] targets; if (node.hasTargets()) { targets = node.getTargets(); } else { targets = PlannerUtil.schemaToTargets(node.getLogicalSchema()); } LinkedHashSet<Target> projectedTargets = Sets.newLinkedHashSet(); for (Iterator<Target> it = getFilteredTarget(targets, newContext.requiredSet); it.hasNext();) { Target target = it.next(); newContext.addExpr(target); } for (Iterator<Target> it = context.targetListMgr.getFilteredTargets(newContext.requiredSet); it.hasNext();) { Target target = it.next(); if (LogicalPlanner.checkIfBeEvaluatedAtRelation(block, target.getEvalTree(), node)) { projectedTargets.add(target); newContext.targetListMgr.markAsEvaluated(target); } } node.setTargets(projectedTargets.toArray(new Target[projectedTargets.size()])); LogicalPlanner.verifyProjectedFields(block, node); return node; }
public static EvalNode getRootSelection(String query) throws TajoException { Expr block = analyzer.parse(query); LogicalPlan plan = planner.createPlan(defaultContext, block); LogicalPlanner.PlanContext context = new LogicalPlanner.PlanContext(defaultContext, plan, plan.getRootBlock(), new EvalTreeOptimizer(), true); Selection selection = plan.getRootBlock().getSingletonExpr(OpType.Filter); return planner.getExprAnnotator().createEvalNode(context, selection.getQual(), NameResolvingMode.RELS_AND_SUBEXPRS); }
EvalNode evalNode = exprAnnotator.createEvalNode(context, rawTarget.getExpr(), NameResolvingMode.RELS_ONLY); if (checkIfBeEvaluatedAtRelation(block, evalNode, scanNode)) { block.namedExprsMgr.markAsEvaluated(rawTarget.getAlias(), evalNode); LinkedHashSet<Target> targets = createFieldTargetsFromRelation(block, scanNode, newlyEvaluatedExprsReferences); verifyProjectedFields(block, scanNode); return scanNode;
LogicalNode child = visit(context, stack, sort.getChild()); if (block.isAggregationRequired()) { child = insertGroupbyNode(context, child, stack); SortSpec[] annotatedSortSpecs = annotateSortSpecs(block, referNames, sortSpecs); if (annotatedSortSpecs.length == 0) { return child;
throws TajoException { stack.push(createIndex); LogicalNode child = visit(context, stack, createIndex.getChild()); stack.pop(); createIndexNode.setKeySortSpecs(relationNode.getLogicalSchema(), annotateSortSpecs(block, referNames, sortSpecs)); createIndexNode.setIndexMethod(IndexMethod.valueOf(createIndex.getMethodSpec().getName().toUpperCase())); if (createIndex.isExternal()) { } else { createIndexNode.setIndexPath( getIndexPath(context, context.queryContext.get(SessionVars.CURRENT_DATABASE), createIndex.getIndexName()));
if (LogicalPlanner.checkIfBeEvaluatedAtJoin(block, fieldReference.getEvalTree(), node, stack.peek().getType() != NodeType.JOIN)) { projectedTargets.add(fieldReference); } else if (LogicalPlanner.checkIfBeEvaluatedAtJoin(block, target.getEvalTree(), node, stack.peek().getType() != NodeType.JOIN)) { projectedTargets.add(target); LogicalPlanner.verifyProjectedFields(block, node); return node;
@Override public LogicalNode visitTableSubQuery(FilterPushDownContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, TableSubQueryNode node, Stack<LogicalNode> stack) throws TajoException { List<EvalNode> matched = new ArrayList<>(); List<EvalNode> unmatched = new ArrayList<>(); for (EvalNode eval : context.pushingDownFilters) { if (LogicalPlanner.checkIfBeEvaluatedAtRelation(block, eval, node)) { matched.add(eval); } else { unmatched.add(eval); } } // transformed -> pushingDownFilters Map<EvalNode, EvalNode> transformedMap = transformEvalsWidthByPassNode(matched, plan, block, node, node.getSubQuery()); context.setFiltersTobePushed(transformedMap.keySet()); visit(context, plan, plan.getBlock(node.getSubQuery())); context.setToOrigin(transformedMap); context.addFiltersTobePushed(unmatched); return node; }
private List<String> getNewlyEvaluatedExprsForJoin(PlanContext context, JoinNode joinNode, boolean isTopMostJoin) throws TajoException { QueryBlock block = context.queryBlock; EvalNode evalNode; List<String> newlyEvaluatedExprs = new ArrayList<>(); for (Iterator<NamedExpr> it = block.namedExprsMgr.getIteratorForUnevaluatedExprs(); it.hasNext();) { NamedExpr namedExpr = it.next(); try { evalNode = exprAnnotator.createEvalNode(context, namedExpr.getExpr(), NameResolvingMode.LEGACY); // the predicates specified in the on clause are already processed in visitJoin() if (LogicalPlanner.checkIfBeEvaluatedAtJoin(context.queryBlock, evalNode, joinNode, isTopMostJoin)) { block.namedExprsMgr.markAsEvaluated(namedExpr.getAlias(), evalNode); newlyEvaluatedExprs.add(namedExpr.getAlias()); } } catch (UndefinedColumnException ve) { } } return newlyEvaluatedExprs; }
LogicalPlanner planner = new LogicalPlanner(catalog, TablespaceManager.getInstance()); LogicalOptimizer optimizer = new LogicalOptimizer(systemConf, catalog, TablespaceManager.getInstance()); Expr expr = JsonHelper.fromJson(jsonExpr, Expr.class); plan = planner.createPlan(queryContext, expr); optimizer.optimize(queryContext, plan);
public LogicalNode visitScan(Context context, LogicalPlan plan, LogicalPlan.QueryBlock block, ScanNode node, Stack<LogicalNode> stack) throws TajoException { Context newContext = new Context(context); List<Target> targets; if (node.hasTargets()) { targets = node.getTargets(); } else { targets = PlannerUtil.schemaToTargets(node.getLogicalSchema()); } LinkedHashSet<Target> projectedTargets = Sets.newLinkedHashSet(); for (Iterator<Target> it = getFilteredTarget(targets, newContext.requiredSet); it.hasNext();) { Target target = it.next(); newContext.addExpr(target); } for (Iterator<Target> it = context.targetListMgr.getFilteredTargets(newContext.requiredSet); it.hasNext();) { Target target = it.next(); if (LogicalPlanner.checkIfBeEvaluatedAtRelation(block, target.getEvalTree(), node)) { projectedTargets.add(target); newContext.targetListMgr.markAsEvaluated(target); } } node.setTargets(new ArrayList<>(projectedTargets)); LogicalPlanner.verifyProjectedFields(block, node); return node; }
EvalNode evalNode = exprAnnotator.createEvalNode(context, rawTarget.getExpr(), NameResolvingMode.RELS_ONLY); if (checkIfBeEvaluatedAtRelation(block, evalNode, scanNode)) { block.namedExprsMgr.markAsEvaluated(rawTarget.getAlias(), evalNode); LinkedHashSet<Target> targets = createFieldTargetsFromRelation(block, scanNode, newlyEvaluatedExprsReferences); verifyProjectedFields(block, scanNode); return scanNode;
private void setTargetOfTableSubQuery (PlanContext context, QueryBlock block, TableSubQueryNode subQueryNode) throws TajoException { // Add additional expressions required in upper nodes. Set<String> newlyEvaluatedExprs = TUtil.newHashSet(); for (NamedExpr rawTarget : block.namedExprsMgr.getAllNamedExprs()) { try { EvalNode evalNode = exprAnnotator.createEvalNode(context, rawTarget.getExpr(), NameResolvingMode.RELS_ONLY); if (checkIfBeEvaluatedAtRelation(block, evalNode, subQueryNode)) { block.namedExprsMgr.markAsEvaluated(rawTarget.getAlias(), evalNode); newlyEvaluatedExprs.add(rawTarget.getAlias()); // newly added exr } } catch (UndefinedColumnException ve) { } } // Assume that each unique expr is evaluated once. LinkedHashSet<Target> targets = createFieldTargetsFromRelation(block, subQueryNode, newlyEvaluatedExprs); for (String newAddedExpr : newlyEvaluatedExprs) { targets.add(block.namedExprsMgr.getTarget(newAddedExpr, true)); } subQueryNode.setTargets(targets.toArray(new Target[targets.size()])); }