public PlanProto.ScanNode.Builder buildScanNode(ScanNode scan) { PlanProto.ScanNode.Builder scanBuilder = PlanProto.ScanNode.newBuilder(); scanBuilder.setTable(scan.getTableDesc().getProto()); if (scan.hasAlias()) { scanBuilder.setAlias(scan.getAlias()); } if (scan.hasTargets()) { scanBuilder.setExistTargets(true); scanBuilder.addAllTargets(ProtoUtil.<PlanProto.Target>toProtoObjects(scan.getTargets().toArray(new ProtoObject[scan.getTargets().size()]))); } else { scanBuilder.setExistTargets(false); } if (scan.hasQual()) { scanBuilder.setQual(EvalNodeSerializer.serialize(scan.getQual())); } scanBuilder.setBroadcast(scan.isBroadcastTable()); scanBuilder.setNameResolveBase(scan.isNameResolveBase()); return scanBuilder; }
public void visitScan(SQLBuilderContext ctx, ScanNode scan, Stack<LogicalNode> stack) { StringBuilder selectClause = new StringBuilder("SELECT "); if (scan.getTargets().length > 0) { selectClause.append(generateTargetList(scan.getTargets())); } else { selectClause.append("1"); } selectClause.append(" "); ctx.sb.append("FROM ").append(scan.getTableName()).append(" "); if (scan.hasAlias()) { ctx.sb.append("AS ").append(scan.getAlias()).append(" "); } if (scan.hasQual()) { ctx.sb.append("WHERE " + sqlExprGen.generate(scan.getQual())); } }
@Override public LogicalNode visitScan(PlanShapeFixerContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, ScanNode node, Stack<LogicalNode> stack) throws TajoException { super.visitScan(context, plan, block, node, stack); context.childNumbers.push(1); if (node.hasTargets()) { node.setTargets(sortTargets(node.getTargets())); } if (node.hasQual()) { node.setQual(sortQual(node.getQual())); } return null; }
public SystemPhysicalExec(TaskAttemptContext context, ScanNode scanNode) { super(context, scanNode.getInSchema(), scanNode.getOutSchema()); this.scanNode = scanNode; if (this.scanNode.hasQual()) { this.qual = this.scanNode.getQual(); this.qual.bind(null, inSchema); } cachedData = new ArrayList<>(); currentRow = 0; isClosed = false; projector = new Projector(context, inSchema, outSchema, scanNode.getTargets()); }
private void updateDescsForScanNodes(NodeType nodeType) { assert nodeType == NodeType.SCAN || nodeType == NodeType.PARTITIONS_SCAN || nodeType == NodeType.INDEX_SCAN; LogicalNode[] scanNodes = PlannerUtil.findAllNodes(plan, nodeType); if (scanNodes != null) { for (LogicalNode node : scanNodes) { ScanNode scanNode = (ScanNode) node; descs.put(scanNode.getCanonicalName(), scanNode.getTableDesc()); } } }
@VisibleForTesting public Path [] findFilteredPartitionPaths(OverridableConf queryContext, ScanNode scanNode) throws IOException { TableDesc table = scanNode.getTableDesc(); PartitionMethodDesc partitionDesc = scanNode.getTableDesc().getPartitionMethod(); if (scanNode.hasQual()) { EvalNode [] conjunctiveForms = AlgebraicUtil.toConjunctiveNormalFormArray(scanNode.getQual()); Set<EvalNode> remainExprs = Sets.newHashSet(conjunctiveForms); paritionValuesSchema.setQualifier(scanNode.getCanonicalName()); for (Column column : paritionValuesSchema.getRootColumns()) { for (EvalNode simpleExpr : conjunctiveForms) { scanNode.setQual(null); } else { scanNode.setQual( AlgebraicUtil.createSingletonExprFromCNF(remainExprs.toArray(new EvalNode[remainExprs.size()])));
@Override public LogicalNode visitScan(Context context, LogicalPlan plan, LogicalPlan.QueryBlock block, ScanNode node, Stack<LogicalNode> stack) throws TajoException { if (node.hasTargets()) { for (Target target : node.getTargets()) { ExprsVerifier.verify(context.state, node, target.getEvalTree()); } } if (node.hasQual()) { ExprsVerifier.verify(context.state, node, node.getQual()); } verifyProjectableOutputSchema(context, node); return node; }
@Override public PlanString getPlanString() { PlanString planStr = new PlanString(this).appendTitle(" on ").appendTitle(getTableName()); if (hasAlias()) { planStr.appendTitle(" as ").appendTitle(alias); } if (hasQual()) { planStr.addExplan("filter: ").appendExplain(this.qual.toString()); } if (hasTargets()) { planStr.addExplan("target list: ").appendExplain(StringUtils.join(targets, ", ")); } planStr.addDetail("out schema: ").appendDetail(getOutSchema().toString()); planStr.addDetail("in schema: ").appendDetail(getInSchema().toString()); return planStr; } }
public String toString() { StringBuilder sb = new StringBuilder("Scan (table=").append(getTableName()); if (hasAlias()) { sb.append(", alias=").append(alias); } if (hasQual()) { sb.append(", filter=").append(qual); } sb.append(", path=").append(getTableDesc().getUri()).append(")"); return sb.toString(); }
private static void fillScanNode(OverridableConf context, EvalContext evalContext, PlanProto.LogicalNode protoNode, ScanNode scan) { PlanProto.ScanNode scanProto = protoNode.getScan(); if (scanProto.hasAlias()) { scan.init(new TableDesc(scanProto.getTable()), scanProto.getAlias()); } else { scan.init(new TableDesc(scanProto.getTable())); } if (scanProto.getExistTargets()) { scan.setTargets(convertTargets(context, evalContext, scanProto.getTargetsList())); } if (scanProto.hasQual()) { scan.setQual(EvalNodeDeserializer.deserialize(context, evalContext, scanProto.getQual())); } if(scanProto.hasBroadcast()){ scan.setBroadcastTable(scanProto.getBroadcast()); } scan.setInSchema(convertSchema(protoNode.getInSchema())); scan.setOutSchema(convertSchema(protoNode.getOutSchema())); scan.setNameResolveBase(scanProto.getNameResolveBase()); }
ExecutionBlockId scanEBId = TajoIdUtils.createExecutionBlockId(scans[i].getTableName()); stats[i] = masterContext.getStage(scanEBId).getResultStats().getNumBytes(); fragments[i] = new FileFragment(scans[i].getCanonicalName(), new Path("/dummy"), 0, 0, new String[]{UNKNOWN_HOST}); List<Fragment> fileFragments = space.getSplits(scans[i].getCanonicalName(), tableDesc, null); if (fileFragments.size() > 0) { fragments[i] = fileFragments.get(0); } else { fragments[i] = new FileFragment(scans[i].getCanonicalName(), new Path(tableDesc.getUri()), 0, 0, new String[]{UNKNOWN_HOST}); LogicalNode rightNode = joinNode.getRightChild(); for (int i = 0; i < stats.length; i++) { if (scans[i].getPID() == leftNode.getPID() || scans[i].getPID() == rightNode.getPID()) { if (stats[i] == 0) { LOG.info(scans[i] + " 's input data is zero. Inner join's result is empty."); if (stats.length == 2) { for (int i = 0; i < stats.length; i++) { if (scans[i].equals(leftScanNode)) { leftStats = stats[i]; } else if (scans[i].equals(rightScanNode)) { rightStats = stats[i]; if (scans[i].getTableDesc().getMeta().getDataFormat().equalsIgnoreCase(intermediateDataFormat)) { nonLeafScanNamesBuilder.append(namePrefix).append(scans[i].getCanonicalName());
public IndexScanNode(int pid, ScanNode scanNode , Schema keySchema , SimplePredicate[] predicates, URI indexPath) { this(pid); init(scanNode.getTableDesc()); setQual(scanNode.getQual()); setInSchema(scanNode.getInSchema()); setTargets(scanNode.getTargets()); this.set(keySchema, predicates, indexPath); }
ScanNode scanNode = new ScanNode(1); scanNode.setQual(evalNodeEq); Tablespace tablespace = TablespaceManager.getByName("cluster1"); List<Fragment> fragments = tablespace.getSplits("hbase_mapped_table", tableDesc, false, scanNode.getQual()); assertEquals(1, fragments.size()); assertEquals("021", ((HBaseFragment)fragments.get(0)).getStartKey().toString()); new ConstEval(new TextDatum("055"))); EvalNode evalNodeA = new BinaryEval(EvalType.AND, evalNode1, evalNode2); scanNode.setQual(evalNodeA); fragments = tablespace.getSplits("hbase_mapped_table", tableDesc, false, scanNode.getQual()); assertEquals(2, fragments.size()); HBaseFragment fragment1 = (HBaseFragment) fragments.get(0); new ConstEval(new TextDatum("075"))); EvalNode evalNodeB = new BinaryEval(EvalType.OR, evalNodeA, evalNode3); scanNode.setQual(evalNodeB); fragments = tablespace.getSplits("hbase_mapped_table", tableDesc, false, scanNode.getQual()); assertEquals(3, fragments.size()); fragment1 = (HBaseFragment) fragments.get(0); EvalNode evalNodeC = new BinaryEval(EvalType.AND, evalNode4, evalNode5); EvalNode evalNodeD = new BinaryEval(EvalType.OR, evalNodeA, evalNodeC); scanNode.setQual(evalNodeD); fragments = tablespace.getSplits("hbase_mapped_table", tableDesc, false, scanNode.getQual()); assertEquals(3, fragments.size());
PartitionMethodDesc partitionDesc = plan.getTableDesc().getPartitionMethod(); Schema columnPartitionSchema = SchemaUtil.clone(partitionDesc.getExpressionSchema()); String qualifier = inSchema.getColumn(0).getQualifier(); this.inSchema = plan.getTableDesc().getSchema(); for (int i = 0; i < plan.getTargets().length; i++) { Target target = plan.getTargets()[i]; if (plan.hasQual()) { EvalTreeUtil.replace(plan.getQual(), targetExpr, constExpr);
public SeqScanExec(TaskAttemptContext context, ScanNode plan, CatalogProtos.FragmentProto [] fragments) throws IOException { super(context, plan.getInSchema(), plan.getOutSchema()); this.plan = plan; this.qual = plan.getQual(); this.fragments = fragments; if (plan.getTableDesc().hasPartition() && plan.getTableDesc().getPartitionMethod().getPartitionType() == CatalogProtos.PartitionType.COLUMN) { rewriteColumnPartitionedTableSchema(); } }
@Override public Object visitScan(Object object, LogicalPlan plan, LogicalPlan.QueryBlock block, ScanNode scanNode, Stack<LogicalNode> stack) throws TajoException { final TableDesc table = scanNode.getTableDesc(); if (!isVirtual(table)) { final TableStats stats = getTableStat(table); final long tableSize = stats.getNumBytes(); final Optional<EvalNode> filter = scanNode.hasQual() ? Optional.of(scanNode.getQual()) : Optional.<EvalNode>absent(); // If USE_TABLE_VOLUME is set, we will update the table volume through a storage handler. // In addition, if the table size is zero, we will update too. // It is a good workaround to avoid suboptimal join orders without cheap cost. if (conf.getBool(SessionVars.USE_TABLE_VOLUME) || tableSize == 0) { table.getStats().setNumBytes(getTableVolume(table, filter)); } } return scanNode; }
@Override public String getTableName() { return plan.getTableName(); }
public LogicalNode visitScan(Context context, LogicalPlan plan, LogicalPlan.QueryBlock block, ScanNode node, Stack<LogicalNode> stack) throws TajoException { Context newContext = new Context(context); Target [] targets; if (node.hasTargets()) { targets = node.getTargets(); } else { targets = PlannerUtil.schemaToTargets(node.getLogicalSchema()); } LinkedHashSet<Target> projectedTargets = Sets.newLinkedHashSet(); for (Iterator<Target> it = getFilteredTarget(targets, newContext.requiredSet); it.hasNext();) { Target target = it.next(); newContext.addExpr(target); } for (Iterator<Target> it = context.targetListMgr.getFilteredTargets(newContext.requiredSet); it.hasNext();) { Target target = it.next(); if (LogicalPlanner.checkIfBeEvaluatedAtRelation(block, target.getEvalTree(), node)) { projectedTargets.add(target); newContext.targetListMgr.markAsEvaluated(target); } } node.setTargets(projectedTargets.toArray(new Target[projectedTargets.size()])); LogicalPlanner.verifyProjectedFields(block, node); return node; }
public Schema getProjectSchema() { Schema projected; // in the case where projected column or expression are given // the target can be an empty list. if (plan.hasTargets()) { projected = new Schema(); Set<Column> columnSet = new HashSet<Column>(); if (plan.hasQual()) { columnSet.addAll(EvalTreeUtil.findUniqueColumns(qual)); } for (Target t : plan.getTargets()) { columnSet.addAll(EvalTreeUtil.findUniqueColumns(t.getEvalTree())); } for (Column column : inSchema.getAllColumns()) { if (columnSet.contains(column)) { projected.addColumn(column); } } } else { // no any projected columns, meaning that all columns should be projected. // TODO - this implicit rule makes code readability bad. So, we should remove it later projected = outSchema; } return projected; }
@Test public final void testBinaryOperator1() throws CloneNotSupportedException, TajoException { QueryContext qc = createQueryContext(); Expr expr = sqlAnalyzer.parse(QUERIES[0]); LogicalPlan plan = planner.createPlan(qc, expr); LogicalOptimizer optimizer = new LogicalOptimizer(util.getConfiguration(), catalog, TablespaceManager.getInstance()); optimizer.optimize(plan); LogicalNode node = plan.getRootBlock().getRoot(); ScanNode scanNode = PlannerUtil.findTopNode(node, NodeType.SCAN); EvalNodeToExprConverter convertor = new EvalNodeToExprConverter(scanNode.getTableName()); convertor.visit(null, scanNode.getQual(), new Stack<>()); Expr resultExpr = convertor.getResult(); BinaryOperator binaryOperator = AlgebraicUtil.findTopExpr(resultExpr, OpType.GreaterThan); assertNotNull(binaryOperator); ColumnReferenceExpr column = binaryOperator.getLeft(); assertEquals("default.lineitem", column.getQualifier()); assertEquals("l_orderkey", column.getName()); LiteralValue literalValue = binaryOperator.getRight(); assertEquals("500", literalValue.getValue()); assertEquals(LiteralValue.LiteralType.Unsigned_Integer, literalValue.getValueType()); }