@VisibleForTesting public LogicalPlan createPlan(OverridableConf queryContext, Expr expr, boolean debug) throws TajoException { LogicalPlan plan = new LogicalPlan(); QueryBlock rootBlock = plan.newAndGetBlock(LogicalPlan.ROOT_BLOCK); PlanContext context = new PlanContext(queryContext, plan, rootBlock, evalOptimizer, debug); preprocessor.process(context, expr); plan.resetGeneratedId(); LogicalNode topMostNode = this.visit(context, new Stack<Expr>(), expr); // Add Root Node LogicalRootNode root = plan.createNode(LogicalRootNode.class); root.setInSchema(topMostNode.getOutSchema()); root.setChild(topMostNode); root.setOutSchema(topMostNode.getOutSchema()); plan.getRootBlock().setRoot(root); return plan; }
/** * Checks whether the query has 'from clause' or not. * * @param plan The logical plan * @return True if a query does not have 'from clause'. */ public static boolean checkIfNonFromQuery(LogicalPlan plan) { LogicalNode node = plan.getRootBlock().getRoot(); // one block, without where clause, no group-by, no-sort, no-join boolean isOneQueryBlock = plan.getQueryBlocks().size() == 1; boolean noRelation = !plan.getRootBlock().hasAlgebraicExpr(OpType.Relation); return !checkIfDDLPlan(node) && noRelation && isOneQueryBlock; }
/** * Adds an expression and returns a reference name. * @param expr added expression */ public String addExpr(Expr expr) { if (idToExprBiMap.inverse().containsKey(expr)) { int refId = idToExprBiMap.inverse().get(expr); return idToNamesMap.get(refId).get(0); } if (block.isRegisteredConst(expr)) { return block.getConstReference(expr); } String generatedName = plan.generateUniqueColumnName(expr); return addExpr(expr, generatedName); }
private TableSubQueryNode visitCommonTableSubquery(PlanContext context, Stack<Expr> stack, CommonSubquery expr) throws TajoException { QueryBlock currentBlock = context.queryBlock; QueryBlock childBlock = context.plan.getBlock(context.plan.getBlockNameByExpr(expr.getSubQuery())); context.plan.connectBlocks(childBlock, currentBlock, BlockType.TableSubQuery); PlanContext newContext = new PlanContext(context, childBlock); context.plan.connectBlocks(childBlock, context.queryBlock, BlockType.TableSubQuery); LogicalNode child = visit(newContext, new Stack<>(), expr.getSubQuery()); TableSubQueryNode subQueryNode = currentBlock.getNodeFromExpr(expr); subQueryNode.setSubQuery(child); setTargetOfTableSubQuery(context, currentBlock, subQueryNode); return subQueryNode; }
@Test public final void testQueryBlockGraph() { LogicalPlan plan = new LogicalPlan(); LogicalPlan.QueryBlock root = plan.newAndGetBlock(LogicalPlan.ROOT_BLOCK); LogicalPlan.QueryBlock new1 = plan.newQueryBlock(); LogicalPlan.QueryBlock new2 = plan.newQueryBlock(); plan.getQueryBlockGraph().addEdge(new1.getName(), root.getName(), new LogicalPlan.BlockEdge(new1, root, BlockType.TableSubQuery)); plan.getQueryBlockGraph().addEdge(new2.getName(), root.getName(), new LogicalPlan.BlockEdge(new2, root, BlockType.TableSubQuery)); SimpleDirectedGraph<String, LogicalPlan.BlockEdge> graph = plan.getQueryBlockGraph(); assertEquals(2, graph.getChildCount(root.getName())); result.add(new2); Set<LogicalPlan.QueryBlock> childs = new HashSet<>(plan.getChildBlocks(root)); assertEquals(result, childs); assertEquals(root, plan.getParentBlock(new1)); assertEquals(root, plan.getParentBlock(new2));
@Override public LogicalNode visitSetSession(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, SetSession expr) throws TajoException { SetSessionNode setSession = ctx.getPlan().createNode(SetSessionNode.class); return setSession; }
public void prepareTest(LogicalPlan plan) throws TajoException { // Plan shape fixer shapeFixerContext.reset(); shapeFixer.visit(shapeFixerContext, plan, plan.getRootBlock()); }
@Override public boolean isEligible(LogicalPlanRewriteRuleContext context) { for (LogicalPlan.QueryBlock block : context.getPlan().getQueryBlocks()) { if (block.hasNode(NodeType.SELECTION) || block.hasNode(NodeType.JOIN)) { return true; } } return false; }
public LogicalNode optimize(OverridableConf context, LogicalPlan plan) throws TajoException { rulesBeforeJoinOpt.rewrite(new LogicalPlanRewriteRuleContext(context, plan, catalog, storage)); DirectedGraphCursor<String, BlockEdge> blockCursor = new DirectedGraphCursor<String, BlockEdge>(plan.getQueryBlockGraph(), plan.getRootBlock().getName()); if (context == null || context.getBool(SessionVars.TEST_JOIN_OPT_ENABLED)) { // default is true while (blockCursor.hasNext()) { optimizeJoinOrder(plan, blockCursor.nextBlock()); } } else { LOG.info("Skip join order optimization"); } rulesAfterToJoinOpt.rewrite(new LogicalPlanRewriteRuleContext(context, plan, catalog, storage)); return plan.getRootBlock().getRoot(); }
while (!plan.getRootBlock().getName().equals(current.getName())) { LogicalPlan.QueryBlock parentBlock = plan.getParentBlock(current); for (RelationNode relationNode : parentBlock.getRelations()) { if (relationNode.getLogicalSchema().containsByQualifiedName(columnRef.getCanonicalName())) {
private SortExec createSortExecForDistinctGroupby(TaskAttemptContext context, DistinctGroupbyNode distinctNode, PhysicalExec subOp, int phase) throws IOException { SortNode sortNode = LogicalPlan.createNodeWithoutPID(SortNode.class); //2 phase: seq, groupby columns, distinct1 keys, distinct2 keys, //3 phase: groupby columns, seq, distinct1 keys, distinct2 keys, List<SortSpec> sortSpecs = new ArrayList<>(); if (phase == 2) { sortSpecs.add(new SortSpec(distinctNode.getTargets().get(0).getNamedColumn())); } for (Column eachColumn: distinctNode.getGroupingColumns()) { sortSpecs.add(new SortSpec(eachColumn)); } if (phase == 3) { sortSpecs.add(new SortSpec(distinctNode.getTargets().get(0).getNamedColumn())); } for (GroupbyNode eachGroupbyNode: distinctNode.getSubPlans()) { for (Column eachColumn: eachGroupbyNode.getGroupingColumns()) { sortSpecs.add(new SortSpec(eachColumn)); } } sortNode.setSortSpecs(sortSpecs.toArray(new SortSpec[sortSpecs.size()])); sortNode.setInSchema(distinctNode.getInSchema()); sortNode.setOutSchema(distinctNode.getInSchema()); ExternalSortExec sortExec = new ExternalSortExec(context, sortNode, subOp); return sortExec; }
@Override public LogicalNode visitTableSubQuery(ProcessorContext ctx, Stack<Expr> stack, TablePrimarySubQuery expr) throws TajoException { QueryBlock childBlock = ctx.planContext.getPlan().getBlock( ctx.planContext.getPlan().getBlockNameByExpr(expr.getSubQuery())); ProcessorContext newContext = new ProcessorContext(new PlanContext(ctx.planContext, childBlock)); super.visitTableSubQuery(newContext, stack, expr); TableSubQueryNode node = getNodeFromExpr(ctx.planContext.getPlan(), expr); LogicalNode child = getNonRelationListExpr(ctx.planContext.getPlan(), expr.getSubQuery()); node.setInSchema(child.getOutSchema()); node.setOutSchema(node.getInSchema()); return node; }
public Schema getOutputSchema() { return getRootNode().getOutSchema(); }
public QueryBlock getBlockByExpr(Expr expr) { return getBlock(exprToBlockNameMap.get(ObjectUtils.identityToString(expr))); }
@Override public LogicalNode visitCreateDatabase(LogicalPlanner.PlanContext ctx, Stack<Expr> stack, CreateDatabase expr) throws TajoException { CreateDatabaseNode createDatabaseNode = ctx.getPlan().createNode(CreateDatabaseNode.class); return createDatabaseNode; }
public VerificationState verify(VerificationState state, LogicalPlan plan) throws TajoException { Context context = new Context(state); visit(context, plan, plan.getRootBlock()); return context.state; }
@Override public boolean isEligible(LogicalPlanRewriteRuleContext context) { for (LogicalPlan.QueryBlock block : context.getPlan().getQueryBlocks()) { if (block.hasNode(NodeType.SELECTION) || block.hasNode(NodeType.JOIN)) { return true; } } return false; }
public LogicalNode optimize(OverridableConf context, LogicalPlan plan) throws TajoException { rulesBeforeJoinOpt.rewrite(new LogicalPlanRewriteRuleContext(context, plan, catalog, storage)); DirectedGraphCursor<String, BlockEdge> blockCursor = new DirectedGraphCursor<>(plan.getQueryBlockGraph(), plan.getRootBlock().getName()); if (context == null || context.getBool(SessionVars.TEST_JOIN_OPT_ENABLED)) { // default is true while (blockCursor.hasNext()) { optimizeJoinOrder(plan, blockCursor.nextBlock()); } } else { LOG.info("Skip join order optimization"); } rulesAfterToJoinOpt.rewrite(new LogicalPlanRewriteRuleContext(context, plan, catalog, storage)); return plan.getRootBlock().getRoot(); }
while (!plan.getRootBlock().getName().equals(current.getName())) { LogicalPlan.QueryBlock parentBlock = plan.getParentBlock(current); for (RelationNode relationNode : parentBlock.getRelations()) { if (relationNode.getLogicalSchema().containsByQualifiedName(columnRef.getCanonicalName())) {
private SortExec createSortExecForDistinctGroupby(TaskAttemptContext context, DistinctGroupbyNode distinctNode, PhysicalExec subOp, int phase) throws IOException { SortNode sortNode = LogicalPlan.createNodeWithoutPID(SortNode.class); //2 phase: seq, groupby columns, distinct1 keys, distinct2 keys, //3 phase: groupby columns, seq, distinct1 keys, distinct2 keys, List<SortSpec> sortSpecs = new ArrayList<SortSpec>(); if (phase == 2) { sortSpecs.add(new SortSpec(distinctNode.getTargets()[0].getNamedColumn())); } for (Column eachColumn: distinctNode.getGroupingColumns()) { sortSpecs.add(new SortSpec(eachColumn)); } if (phase == 3) { sortSpecs.add(new SortSpec(distinctNode.getTargets()[0].getNamedColumn())); } for (GroupbyNode eachGroupbyNode: distinctNode.getSubPlans()) { for (Column eachColumn: eachGroupbyNode.getGroupingColumns()) { sortSpecs.add(new SortSpec(eachColumn)); } } sortNode.setSortSpecs(sortSpecs.toArray(new SortSpec[]{})); sortNode.setInSchema(distinctNode.getInSchema()); sortNode.setOutSchema(distinctNode.getInSchema()); ExternalSortExec sortExec = new ExternalSortExec(context, sortNode, subOp); return sortExec; }