private void optimizeJoinOrder(LogicalPlan plan, String blockName) throws TajoException { LogicalPlan.QueryBlock block = plan.getBlock(blockName); if (block.hasNode(NodeType.JOIN)) { String originalOrder = JoinOrderStringBuilder.buildJoinOrderString(plan, block); double nonOptimizedJoinCost = JoinCostComputer.computeCost(plan, block); // finding relations and filter expressions JoinGraphContext joinGraphContext = JoinGraphBuilder.buildJoinGraph(plan, block); // finding join order and restore remaining filters FoundJoinOrder order = joinOrderAlgorithm.findBestOrder(plan, block, joinGraphContext); // replace join node with FoundJoinOrder. JoinNode newJoinNode = order.getOrderedJoin(); LogicalNode newNode = handleRemainingFiltersIfNecessary(joinGraphContext, plan, block, newJoinNode); JoinNode old = PlannerUtil.findTopNode(block.getRoot(), NodeType.JOIN); JoinTargetCollector collector = new JoinTargetCollector(); Set<Target> targets = new LinkedHashSet<Target>(); collector.visitJoin(targets, plan, block, old, new Stack<LogicalNode>()); if (targets.size() == 0) { newJoinNode.setTargets(PlannerUtil.schemaToTargets(old.getOutSchema())); } else { newJoinNode.setTargets(targets.toArray(new Target[targets.size()])); } PlannerUtil.replaceNode(plan, block.getRoot(), old, newNode); // End of replacement logic String optimizedOrder = JoinOrderStringBuilder.buildJoinOrderString(plan, block); block.addPlanHistory("Non-optimized join order: " + originalOrder + " (cost: " + nonOptimizedJoinCost + ")"); block.addPlanHistory("Optimized join order : " + optimizedOrder + " (cost: " + order.getCost() + ")"); joinGraphContext.clear(); } }
private void optimizeJoinOrder(LogicalPlan plan, String blockName) throws TajoException { LogicalPlan.QueryBlock block = plan.getBlock(blockName); if (block.hasNode(NodeType.JOIN)) { String originalOrder = JoinOrderStringBuilder.buildJoinOrderString(plan, block); double nonOptimizedJoinCost = JoinCostComputer.computeCost(plan, block); // finding relations and filter expressions JoinGraphContext joinGraphContext = JoinGraphBuilder.buildJoinGraph(plan, block); // finding join order and restore remaining filters FoundJoinOrder order = joinOrderAlgorithm.findBestOrder(plan, block, joinGraphContext); // replace join node with FoundJoinOrder. JoinNode newJoinNode = order.getOrderedJoin(); LogicalNode newNode = handleRemainingFiltersIfNecessary(joinGraphContext, plan, block, newJoinNode); JoinNode old = PlannerUtil.findTopNode(block.getRoot(), NodeType.JOIN); JoinTargetCollector collector = new JoinTargetCollector(); Set<Target> targets = new LinkedHashSet<>(); collector.visitJoin(targets, plan, block, old, new Stack<>()); if (targets.size() == 0) { newJoinNode.setTargets(PlannerUtil.schemaToTargets(old.getOutSchema())); } else { newJoinNode.setTargets(new ArrayList<>(targets)); } PlannerUtil.replaceNode(plan, block.getRoot(), old, newNode); // End of replacement logic String optimizedOrder = JoinOrderStringBuilder.buildJoinOrderString(plan, block); block.addPlanHistory("Non-optimized join order: " + originalOrder + " (cost: " + nonOptimizedJoinCost + ")"); block.addPlanHistory("Optimized join order : " + optimizedOrder + " (cost: " + order.getCost() + ")"); joinGraphContext.clear(); } }
/** * Checks whether the target of this query is a virtual table or not. * It will be removed after tajo storage supports catalog service access. * */ public static boolean checkIfQueryTargetIsVirtualTable(LogicalPlan plan) { LogicalRootNode rootNode = plan.getRootBlock().getRoot(); boolean hasScanNode = plan.getRootBlock().hasNode(NodeType.SCAN); LogicalNode[] scanNodes = findAllNodes(rootNode, NodeType.SCAN); boolean isVirtualTable = scanNodes.length > 0; ScanNode scanNode = null; for (LogicalNode node: scanNodes) { scanNode = (ScanNode) node; isVirtualTable &= (scanNode.getTableDesc().getMeta().getDataFormat().equalsIgnoreCase("SYSTEM")); } return !checkIfDDLPlan(rootNode) && hasScanNode && isVirtualTable; }
} else if (functionType == FunctionType.AGGREGATION || functionType == FunctionType.UDA) { if (!ctx.currentBlock.hasNode(NodeType.GROUP_BY)) { ctx.currentBlock.setAggregationRequire();
projectionNode.setOutSchema(PlannerUtil.targetToSchema(targets)); if (projection.isDistinct() && block.hasNode(NodeType.GROUP_BY)) { throw makeSyntaxError("Cannot support grouping and distinct at the same time yet"); } else {
boolean simpleOperator = rootNode.getChild().getType() == NodeType.LIMIT || rootNode.getChild().getType() == NodeType.SCAN || rootNode.getChild().getType() == NodeType.PARTITIONS_SCAN; boolean noOrderBy = !plan.getRootBlock().hasNode(NodeType.SORT); boolean noGroupBy = !plan.getRootBlock().hasNode(NodeType.GROUP_BY); boolean noWhere = !plan.getRootBlock().hasNode(NodeType.SELECTION); boolean noJoin = !plan.getRootBlock().hasNode(NodeType.JOIN); boolean singleRelation = (plan.getRootBlock().hasNode(NodeType.SCAN) || plan.getRootBlock().hasNode(NodeType.PARTITIONS_SCAN)) && PlannerUtil.getRelationLineage(plan.getRootBlock().getRoot()).length == 1;
boolean simpleOperator = rootNode.getChild().getType() == NodeType.LIMIT || rootNode.getChild().getType() == NodeType.SCAN || rootNode.getChild().getType() == NodeType.PARTITIONS_SCAN; boolean noOrderBy = !plan.getRootBlock().hasNode(NodeType.SORT); boolean noGroupBy = !plan.getRootBlock().hasNode(NodeType.GROUP_BY); boolean noWhere = !plan.getRootBlock().hasNode(NodeType.SELECTION); boolean noJoin = !plan.getRootBlock().hasNode(NodeType.JOIN); boolean singleRelation = (plan.getRootBlock().hasNode(NodeType.SCAN) || plan.getRootBlock().hasNode(NodeType.PARTITIONS_SCAN)) && PlannerUtil.getRelationLineage(plan.getRootBlock().getRoot()).length == 1;
if (plan.getRootBlock().hasNode(NodeType.LIMIT)) { LimitNode limitNode = plan.getRootBlock().getNode(NodeType.LIMIT); maxRow = (int) limitNode.getFetchFirstNum();
public void execSimpleQuery(QueryContext queryContext, Session session, String query, LogicalPlan plan, SubmitQueryResponse.Builder response) throws Exception { ScanNode scanNode = plan.getRootBlock().getNode(NodeType.SCAN); final TableDesc table = scanNode.getTableDesc(); if (table.hasPartition()) { scanNode = plan.getRootBlock().getNode(NodeType.PARTITIONS_SCAN); } final TableDesc resultDesc = new TableDesc("", scanNode.getOutSchema(), new TableMeta(BuiltinStorages.DRAW, table.getMeta().getOptions()), null); // push down limit int maxRow = Integer.MAX_VALUE; if (plan.getRootBlock().hasNode(NodeType.LIMIT)) { LimitNode limitNode = plan.getRootBlock().getNode(NodeType.LIMIT); maxRow = (int) limitNode.getFetchFirstNum(); scanNode.setLimit(maxRow); } final QueryInfo queryInfo = context.getQueryJobManager().createNewSimpleQuery(queryContext, session, query, (LogicalRootNode) plan.getRootBlock().getRoot()); final NonForwardQueryResultScanner queryResultScanner = new NonForwardQueryResultFileScanner( context.getConf(), session.getSessionId(), queryInfo.getQueryId(), scanNode, maxRow); queryResultScanner.init(); session.addNonForwardQueryResultScanner(queryResultScanner); response.setState(OK); response.setQueryId(queryInfo.getQueryId().getProto()); response.setResultType(ResultType.ENCLOSED); response.setMaxRowNum(maxRow); response.setTableDesc(resultDesc.getProto()); }
@Override public EvalNode visitGeneralSetFunction(Context ctx, Stack<Expr> stack, GeneralSetFunctionExpr setFunction) throws TajoException { Expr[] params = setFunction.getParams(); EvalNode[] givenArgs = new EvalNode[params.length]; DataType[] paramTypes = new DataType[params.length]; FunctionType functionType = setFunction.isDistinct() ? FunctionType.DISTINCT_AGGREGATION : FunctionType.AGGREGATION; givenArgs[0] = visit(ctx, stack, params[0]); if (setFunction.getSignature().equalsIgnoreCase("count")) { paramTypes[0] = CatalogUtil.newSimpleDataType(TajoDataTypes.Type.ANY); } else { paramTypes[0] = convert(givenArgs[0].getValueType()).getDataType(); } if (!catalog.containFunction(setFunction.getSignature(), functionType, paramTypes)) { throw new UndefinedFunctionException(buildSimpleFunctionSignature(setFunction.getSignature(), paramTypes)); } FunctionDesc funcDesc = catalog.getFunction(setFunction.getSignature(), functionType, paramTypes); if (!ctx.currentBlock.hasNode(NodeType.GROUP_BY)) { ctx.currentBlock.setAggregationRequire(); } return new AggregationFunctionCallEval(funcDesc, givenArgs); }
public void execQueryOnVirtualTable(QueryContext queryContext, Session session, String query, LogicalPlan plan, SubmitQueryResponse.Builder response) throws Exception { int maxRow = Integer.MAX_VALUE; if (plan.getRootBlock().hasNode(NodeType.LIMIT)) { LimitNode limitNode = plan.getRootBlock().getNode(NodeType.LIMIT); maxRow = (int) limitNode.getFetchFirstNum(); } QueryInfo queryInfo = context.getQueryJobManager().createNewSimpleQuery(queryContext, session, query, (LogicalRootNode) plan.getRootBlock().getRoot()); NonForwardQueryResultScanner queryResultScanner = new NonForwardQueryResultSystemScanner( context, plan, queryInfo.getQueryId(), session.getSessionId(), maxRow); queryResultScanner.init(); session.addNonForwardQueryResultScanner(queryResultScanner); response.setState(OK); response.setQueryId(queryInfo.getQueryId().getProto()); response.setResultType(ResultType.ENCLOSED); response.setMaxRowNum(maxRow); response.setTableDesc(queryResultScanner.getTableDesc().getProto()); }
@Override public EvalNode visitGeneralSetFunction(Context ctx, Stack<Expr> stack, GeneralSetFunctionExpr setFunction) throws TajoException { Expr[] params = setFunction.getParams(); EvalNode[] givenArgs = new EvalNode[params.length]; DataType[] paramTypes = new DataType[params.length]; FunctionType functionType = setFunction.isDistinct() ? FunctionType.DISTINCT_AGGREGATION : FunctionType.AGGREGATION; givenArgs[0] = visit(ctx, stack, params[0]); if (setFunction.getSignature().equalsIgnoreCase("count")) { paramTypes[0] = CatalogUtil.newSimpleDataType(Type.ANY); } else { paramTypes[0] = givenArgs[0].getValueType(); } if (!catalog.containFunction(setFunction.getSignature(), functionType, paramTypes)) { throw new UndefinedFunctionException(buildSimpleFunctionSignature(setFunction.getSignature(), paramTypes)); } FunctionDesc funcDesc = catalog.getFunction(setFunction.getSignature(), functionType, paramTypes); if (!ctx.currentBlock.hasNode(NodeType.GROUP_BY)) { ctx.currentBlock.setAggregationRequire(); } return new AggregationFunctionCallEval(funcDesc, givenArgs); }
public void execQueryOnVirtualTable(QueryContext queryContext, Session session, String query, LogicalPlan plan, SubmitQueryResponse.Builder response) throws Exception { int maxRow = Integer.MAX_VALUE; if (plan.getRootBlock().hasNode(NodeType.LIMIT)) { LimitNode limitNode = plan.getRootBlock().getNode(NodeType.LIMIT); maxRow = (int) limitNode.getFetchFirstNum(); } QueryInfo queryInfo = context.getQueryJobManager().createNewSimpleQuery(queryContext, session, query, (LogicalRootNode) plan.getRootBlock().getRoot()); NonForwardQueryResultScanner queryResultScanner = new NonForwardQueryResultSystemScanner( context, plan, queryInfo.getQueryId(), session.getSessionId(), maxRow); queryResultScanner.init(); session.addNonForwardQueryResultScanner(queryResultScanner); response.setState(OK); response.setQueryId(queryInfo.getQueryId().getProto()); response.setResultType(ResultType.ENCLOSED); response.setMaxRowNum(maxRow); response.setTableDesc(queryResultScanner.getTableDesc().getProto()); }
@Override public boolean isEligible(OverridableConf queryContext, MasterPlan plan) { long thresholdForNonCrossJoin = queryContext.getLong(SessionVars.BROADCAST_NON_CROSS_JOIN_THRESHOLD) * StorageUnit.KB; long thresholdForCrossJoin = queryContext.getLong(SessionVars.BROADCAST_CROSS_JOIN_THRESHOLD) * StorageUnit.KB; boolean broadcastJoinEnabled = queryContext.getBool(SessionVars.TEST_BROADCAST_JOIN_ENABLED); if (broadcastJoinEnabled && (thresholdForNonCrossJoin > 0 || thresholdForCrossJoin > 0)) { for (LogicalPlan.QueryBlock block : plan.getLogicalPlan().getQueryBlocks()) { if (block.hasNode(NodeType.JOIN)) { init(plan, thresholdForNonCrossJoin, thresholdForCrossJoin, broadcastJoinEnabled); return true; } } } return false; }
/** * Checks whether the target of this query is a virtual table or not. * It will be removed after tajo storage supports catalog service access. * */ public static boolean checkIfQueryTargetIsVirtualTable(LogicalPlan plan) { LogicalRootNode rootNode = plan.getRootBlock().getRoot(); boolean hasScanNode = plan.getRootBlock().hasNode(NodeType.SCAN); LogicalNode[] scanNodes = findAllNodes(rootNode, NodeType.SCAN); boolean isVirtualTable = scanNodes.length > 0; ScanNode scanNode = null; for (LogicalNode node: scanNodes) { scanNode = (ScanNode) node; isVirtualTable &= (scanNode.getTableDesc().getMeta().getDataFormat().equalsIgnoreCase("SYSTEM")); } return !checkIfDDLPlan(rootNode) && hasScanNode && isVirtualTable; }
@Override public boolean isEligible(OverridableConf queryContext, MasterPlan plan) { long thresholdForNonCrossJoin = queryContext.getLong(SessionVars.BROADCAST_NON_CROSS_JOIN_THRESHOLD) * StorageUnit.KB; long thresholdForCrossJoin = queryContext.getLong(SessionVars.BROADCAST_CROSS_JOIN_THRESHOLD) * StorageUnit.KB; boolean broadcastJoinEnabled = queryContext.getBool(SessionVars.TEST_BROADCAST_JOIN_ENABLED); if (broadcastJoinEnabled && (thresholdForNonCrossJoin > 0 || thresholdForCrossJoin > 0)) { for (LogicalPlan.QueryBlock block : plan.getLogicalPlan().getQueryBlocks()) { if (block.hasNode(NodeType.JOIN)) { init(plan); return true; } } } return false; }
@Override public boolean isEligible(LogicalPlanRewriteRuleContext context) { for (LogicalPlan.QueryBlock block : context.getPlan().getQueryBlocks()) { if (block.hasNode(NodeType.SELECTION) || block.hasNode(NodeType.JOIN)) { rewriter = new Rewriter(context.getPlan()); return true; } } return false; }
@Override public boolean isEligible(LogicalPlanRewriteRuleContext context) { for (LogicalPlan.QueryBlock block : context.getPlan().getQueryBlocks()) { if (block.hasNode(NodeType.SELECTION) || block.hasNode(NodeType.JOIN)) { rewriter = new Rewriter(context.getPlan()); return true; } } return false; }