@VisibleForTesting public LogicalNode optimize(LogicalPlan plan) throws TajoException { OverridableConf conf = new OverridableConf(new TajoConf(), ConfigKey.ConfigType.SESSION, ConfigKey.ConfigType.QUERY, ConfigKey.ConfigType.SYSTEM); return optimize(conf, plan); }
@VisibleForTesting public LogicalNode optimize(LogicalPlan plan) throws TajoException { OverridableConf conf = new OverridableConf(new TajoConf(), ConfigKey.ConfigType.SESSION, ConfigKey.ConfigType.QUERY, ConfigKey.ConfigType.SYSTEM); return optimize(conf, plan); }
@Test public final void testProjectionPushWithInnerJoin() throws TajoException { // two relations Expr expr = sqlAnalyzer.parse(QUERIES[5]); LogicalPlan newPlan = planner.createPlan(defaultContext, expr); optimizer.optimize(newPlan); }
@Test public final void testInsertInto() throws CloneNotSupportedException, TajoException { Expr expr = sqlAnalyzer.parse(TestLogicalPlanner.insertStatements[0]); LogicalPlan newPlan = planner.createPlan(defaultContext, expr); optimizer.optimize(newPlan); }
private static VerificationState verify(String query) throws TajoException { VerificationState state = new VerificationState(); QueryContext context = LocalTajoTestingUtility.createDummyContext(conf); Expr expr = sqlParser.parse(query); verifier.verify(context, state, expr); if (state.getErrors().size() > 0) { return state; } LogicalPlan plan = planner.createPlan(context, expr); optimizer.optimize(plan); postVerifier.verify(state, plan); return state; }
@Test public final void testEnforceForDefaultColumnPartitionStorePlan() throws IOException, TajoException { FileFragment[] frags = FileTablespace.splitNG(conf, "default.score", score.getMeta(), new Path(score.getUri()), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testStorePlan"); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newTaskAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); ctx.setOutputPath(new Path(workDir, "grouped3")); Expr context = analyzer.parse(CreateTableAsStmts[2]); LogicalPlan plan = planner.createPlan(defaultContext, context); LogicalNode rootNode = optimizer.optimize(plan); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); exec.close(); assertTrue(exec instanceof SortBasedColPartitionStoreExec); }
@Benchmark @BenchmarkMode(Mode.All) public void timSort(BenchContext context) throws InterruptedException, IOException, TajoException { QueryContext queryContext = LocalTajoTestingUtility.createDummyContext(conf); queryContext.setInt(SessionVars.EXTSORT_BUFFER_SIZE, 200); queryContext.set(SessionVars.SORT_ALGORITHM.keyname(), "TIM"); FileFragment[] frags = FileTablespace.splitNG(conf, "default.employee", employee.getMeta(), new Path(employee.getUri()), Integer.MAX_VALUE); Path workDir = new Path(testDir, TestExternalSortExec.class.getName()); TaskAttemptContext ctx = new TaskAttemptContext(queryContext, LocalTajoTestingUtility.newTaskAttemptId(), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[0]); LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), expr); LogicalNode rootNode = optimizer.optimize(plan); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); exec.init(); while (exec.next() != null) {} exec.close(); }
@Benchmark @BenchmarkMode(Mode.All) public void msdRadixSort(BenchContext context) throws InterruptedException, IOException, TajoException { QueryContext queryContext = LocalTajoTestingUtility.createDummyContext(conf); queryContext.setInt(SessionVars.EXTSORT_BUFFER_SIZE, 200); queryContext.set(SessionVars.SORT_ALGORITHM.keyname(), "MSD_RADIX"); FileFragment[] frags = FileTablespace.splitNG(conf, "default.employee", employee.getMeta(), new Path(employee.getUri()), Integer.MAX_VALUE); Path workDir = new Path(testDir, TestExternalSortExec.class.getName()); TaskAttemptContext ctx = new TaskAttemptContext(queryContext, LocalTajoTestingUtility.newTaskAttemptId(), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); Expr expr = analyzer.parse(QUERIES[0]); LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), expr); LogicalNode rootNode = optimizer.optimize(plan); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); exec.init(); while (exec.next() != null) {} exec.close(); }
@Test public final void testEnforceForHashBasedColumnPartitionStorePlan() throws IOException, TajoException { Expr context = analyzer.parse(CreateTableAsStmts[2]); LogicalPlan plan = planner.createPlan(defaultContext, context); LogicalRootNode rootNode = (LogicalRootNode) optimizer.optimize(plan); CreateTableNode createTableNode = rootNode.getChild(); Enforcer enforcer = new Enforcer(); enforcer.enforceColumnPartitionAlgorithm(createTableNode.getPID(), ColumnPartitionAlgorithm.HASH_PARTITION); FileFragment[] frags = FileTablespace.splitNG(conf, "default.score", score.getMeta(), new Path(score.getUri()), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testStorePlan"); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newTaskAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(enforcer); ctx.setOutputPath(new Path(workDir, "grouped4")); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); exec.close(); assertTrue(exec instanceof HashBasedColPartitionStoreExec); }
@Test public final void testEnforceForSortBasedColumnPartitionStorePlan() throws IOException, TajoException { Expr context = analyzer.parse(CreateTableAsStmts[2]); LogicalPlan plan = planner.createPlan(defaultContext, context); LogicalRootNode rootNode = (LogicalRootNode) optimizer.optimize(plan); CreateTableNode createTableNode = rootNode.getChild(); Enforcer enforcer = new Enforcer(); enforcer.enforceColumnPartitionAlgorithm(createTableNode.getPID(), ColumnPartitionAlgorithm.SORT_PARTITION); FileFragment[] frags = FileTablespace.splitNG(conf, "default.score", score.getMeta(), new Path(score.getUri()), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testStorePlan"); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newTaskAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(enforcer); ctx.setOutputPath(new Path(workDir, "grouped5")); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); exec.close(); assertTrue(exec instanceof SortBasedColPartitionStoreExec); }
private void isSimpleQuery(String queryStr, boolean expected) throws Exception { GlobalEngine globalEngine = testingCluster.getMaster().getContext().getGlobalEngine(); QueryContext queryContext = LocalTajoTestingUtility.createDummyContext(conf); Session session = testingCluster.getMaster().getContext().getSessionManager().getSession(client.getSessionId()); LogicalPlan plan = globalEngine.getLogicalPlanner(). createPlan(queryContext, globalEngine.buildExpressionFromSql(queryStr, session)); globalEngine.getLogicalOptimizer().optimize(plan); assertEquals(expected, PlannerUtil.checkIfSimpleQuery(plan)); }
@Test public final void testGroupByWithNullValue() throws IOException, TajoException { FileFragment[] frags = FileTablespace.splitNG(conf, "default.score", score.getMeta(), new Path(score.getUri()), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testGroupByWithNullValue"); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newTaskAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[11]); LogicalPlan plan = planner.createPlan(defaultContext, context); LogicalNode rootNode = optimizer.optimize(plan); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); int count = 0; exec.init(); while(exec.next() != null) { count++; } exec.close(); assertEquals(10, count); }
@Test public final void testBinaryOperator1() throws CloneNotSupportedException, TajoException { QueryContext qc = createQueryContext(); Expr expr = sqlAnalyzer.parse(QUERIES[0]); LogicalPlan plan = planner.createPlan(qc, expr); LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog, TablespaceManager.getInstance()); optimizer.optimize(plan); LogicalNode node = plan.getRootBlock().getRoot(); ScanNode scanNode = PlannerUtil.findTopNode(node, NodeType.SCAN); EvalNodeToExprConverter convertor = new EvalNodeToExprConverter(scanNode.getTableName()); convertor.visit(null, scanNode.getQual(), new Stack<>()); Expr resultExpr = convertor.getResult(); BinaryOperator binaryOperator = AlgebraicUtil.findTopExpr(resultExpr, OpType.GreaterThan); assertNotNull(binaryOperator); ColumnReferenceExpr column = binaryOperator.getLeft(); assertEquals("default.lineitem", column.getQualifier()); assertEquals("l_orderkey", column.getName()); LiteralValue literalValue = binaryOperator.getRight(); assertEquals("500", literalValue.getValue()); assertEquals(LiteralValue.LiteralType.Unsigned_Integer, literalValue.getValueType()); }
@Test public final void testBinaryOperator2() throws CloneNotSupportedException, TajoException { QueryContext qc = createQueryContext(); Expr expr = sqlAnalyzer.parse(QUERIES[1]); LogicalPlan plan = planner.createPlan(qc, expr); LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog, TablespaceManager.getInstance()); optimizer.optimize(plan); LogicalNode node = plan.getRootBlock().getRoot(); ScanNode scanNode = PlannerUtil.findTopNode(node, NodeType.SCAN); EvalNodeToExprConverter convertor = new EvalNodeToExprConverter(scanNode.getTableName()); convertor.visit(null, scanNode.getQual(), new Stack<>()); Expr resultExpr = convertor.getResult(); BinaryOperator equals = AlgebraicUtil.findTopExpr(resultExpr, OpType.Equals); assertNotNull(equals); ColumnReferenceExpr column = equals.getLeft(); assertEquals("default.region", column.getQualifier()); assertEquals("r_name", column.getName()); LiteralValue literalValue = equals.getRight(); assertEquals("EUROPE", literalValue.getValue()); assertEquals(LiteralValue.LiteralType.String, literalValue.getValueType()); }
@Test public final void testCountFunction() throws IOException, TajoException { FileFragment[] frags = FileTablespace.splitNG(conf, "default.score", score.getMeta(), new Path(score.getUri()), Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/testCountFunction"); TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), LocalTajoTestingUtility.newTaskAttemptId(masterPlan), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[9]); LogicalPlan plan = planner.createPlan(defaultContext, context); LogicalNode rootNode = optimizer.optimize(plan); // Set all aggregation functions to the first phase mode GroupbyNode groupbyNode = PlannerUtil.findTopNode(rootNode, NodeType.GROUP_BY); for (AggregationFunctionCallEval function : groupbyNode.getAggFunctions()) { function.setFirstPhase(); } PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); exec.init(); Tuple tuple = exec.next(); assertEquals(30, tuple.getInt8(0)); assertNull(exec.next()); exec.close(); }
@Test public final void testNext() throws IOException, TajoException { FileFragment[] frags = FileTablespace.splitNG(conf, "default.employee", employeeMeta, tablePath, Integer.MAX_VALUE); Path workDir = CommonTestingUtil.getTestDir(TajoTestingCluster.DEFAULT_TEST_DIRECTORY + "/TestSortExec"); TaskAttemptContext ctx = new TaskAttemptContext(queryContext, LocalTajoTestingUtility .newTaskAttemptId(), new FileFragment[] { frags[0] }, workDir); ctx.setEnforcer(new Enforcer()); Expr context = analyzer.parse(QUERIES[0]); LogicalPlan plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf), context); LogicalNode rootNode = optimizer.optimize(plan); PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf); PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode); Tuple tuple; Datum preVal = null; Datum curVal; exec.init(); while ((tuple = exec.next()) != null) { curVal = tuple.asDatum(0); if (preVal != null) { assertTrue(preVal.lessThanEqual(curVal).asBool()); } preVal = curVal; } exec.close(); }
@Test public final void testProjectionPush() throws CloneNotSupportedException, TajoException { // two relations Expr expr = sqlAnalyzer.parse(QUERIES[2]); LogicalPlan newPlan = planner.createPlan(defaultContext, expr); LogicalNode plan = newPlan.getRootBlock().getRoot(); assertEquals(NodeType.ROOT, plan.getType()); LogicalRootNode root = (LogicalRootNode) plan; TestLogicalPlanner.testCloneLogicalNode(root); assertEquals(NodeType.PROJECTION, root.getChild().getType()); ProjectionNode projNode = root.getChild(); assertEquals(NodeType.SELECTION, projNode.getChild().getType()); SelectionNode selNode = projNode.getChild(); assertEquals(NodeType.SCAN, selNode.getChild().getType()); LogicalNode optimized = optimizer.optimize(newPlan); assertEquals(NodeType.ROOT, optimized.getType()); root = (LogicalRootNode) optimized; TestLogicalPlanner.testCloneLogicalNode(root); assertEquals(NodeType.SCAN, root.getChild().getType()); }
@Test public final void testCheckingInfinityJoinScore() throws Exception { // Test for TAJO-1552 String query = "select a.deptname from large_dept1 a, large_dept2 b, large_dept3 c, " + "large_dept4 d, large_dept5 e, large_dept6 f "; Expr expr = sqlAnalyzer.parse(query); LogicalPlan newPlan = planner.createPlan(defaultContext, expr); LogicalNode[] joinNodes = PlannerUtil.findAllNodes(newPlan. getRootBlock().getRoot() , NodeType.JOIN); assertNotNull(joinNodes); assertEquals(5, joinNodes.length); assertJoinNode(joinNodes[0], "default.a", "default.b"); assertJoinNode(joinNodes[1], null, "default.c"); assertJoinNode(joinNodes[2], null, "default.d"); assertJoinNode(joinNodes[3], null, "default.e"); assertJoinNode(joinNodes[4], null, "default.f"); optimizer.optimize(newPlan); joinNodes = PlannerUtil.findAllNodes(newPlan. getRootBlock().getRoot() , NodeType.JOIN); assertNotNull(joinNodes); assertEquals(5, joinNodes.length); assertJoinNode(joinNodes[0], "default.d", "default.c"); assertJoinNode(joinNodes[1], "default.b", "default.a"); assertJoinNode(joinNodes[2], null, null); assertJoinNode(joinNodes[3], "default.f", "default.e"); assertJoinNode(joinNodes[4], null, null); }
@Test public final void testOptimizeWithGroupBy() throws CloneNotSupportedException, TajoException { Expr expr = sqlAnalyzer.parse(QUERIES[3]); LogicalPlan newPlan = planner.createPlan(defaultContext, expr); LogicalNode plan = newPlan.getRootBlock().getRoot(); assertEquals(NodeType.ROOT, plan.getType()); LogicalRootNode root = (LogicalRootNode) plan; TestLogicalPlanner.testCloneLogicalNode(root); assertEquals(NodeType.PROJECTION, root.getChild().getType()); ProjectionNode projNode = root.getChild(); assertEquals(NodeType.GROUP_BY, projNode.getChild().getType()); GroupbyNode groupbyNode = projNode.getChild(); assertEquals(NodeType.SELECTION, groupbyNode.getChild().getType()); SelectionNode selNode = groupbyNode.getChild(); assertEquals(NodeType.SCAN, selNode.getChild().getType()); LogicalNode optimized = optimizer.optimize(newPlan); assertEquals(NodeType.ROOT, optimized.getType()); root = (LogicalRootNode) optimized; TestLogicalPlanner.testCloneLogicalNode(root); assertEquals(NodeType.GROUP_BY, root.getChild().getType()); groupbyNode = root.getChild(); assertEquals(NodeType.SCAN, groupbyNode.getChild().getType()); }
@Test public final void testProjectionPushWithNaturalJoin() throws TajoException, CloneNotSupportedException { // two relations Expr expr = sqlAnalyzer.parse(QUERIES[4]); LogicalPlan newPlan = planner.createPlan(defaultContext, expr); LogicalNode plan = newPlan.getRootBlock().getRoot(); assertEquals(NodeType.ROOT, plan.getType()); LogicalRootNode root = (LogicalRootNode) plan; TestLogicalPlanner.testCloneLogicalNode(root); assertEquals(NodeType.PROJECTION, root.getChild().getType()); ProjectionNode projNode = root.getChild(); assertEquals(NodeType.JOIN, projNode.getChild().getType()); JoinNode joinNode = projNode.getChild(); assertEquals(NodeType.SCAN, joinNode.getLeftChild().getType()); assertEquals(NodeType.SCAN, joinNode.getRightChild().getType()); LogicalNode optimized = optimizer.optimize(newPlan); assertEquals(NodeType.ROOT, optimized.getType()); root = (LogicalRootNode) optimized; TestLogicalPlanner.testCloneLogicalNode(root); assertEquals(NodeType.JOIN, root.getChild().getType()); joinNode = root.getChild(); assertEquals(NodeType.SCAN, joinNode.getLeftChild().getType()); assertEquals(NodeType.SCAN, joinNode.getRightChild().getType()); }