@Override public LogicalNode visitPartitionedTableScan(PlanShapeFixerContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, PartitionedTableScanNode node, Stack<LogicalNode> stack) throws TajoException { super.visitPartitionedTableScan(context, plan, block, node, stack); context.childNumbers.push(1); Path[] inputPaths = node.getInputPaths(); Arrays.sort(inputPaths); node.setInputPaths(inputPaths); if (node.hasTargets()) { node.setTargets(sortTargets(node.getTargets())); } if (node.hasQual()) { node.setQual(sortQual(node.getQual())); } return null; }
private void updateTableStat(OverridableConf queryContext, PartitionedTableScanNode scanNode) throws TajoException { if (scanNode.getInputPaths().length > 0) { try { FileSystem fs = scanNode.getInputPaths()[0].getFileSystem(queryContext.getConf()); long totalVolume = 0; for (Path input : scanNode.getInputPaths()) { ContentSummary summary = fs.getContentSummary(input); totalVolume += summary.getLength(); } scanNode.getTableDesc().getStats().setNumBytes(totalVolume); } catch (Throwable e) { throw new TajoInternalError(e); } } }
private static PartitionedTableScanNode convertPartitionScan(OverridableConf context, EvalContext evalContext, PlanProto.LogicalNode protoNode) { PartitionedTableScanNode partitionedScan = new PartitionedTableScanNode(protoNode.getNodeId()); fillScanNode(context, evalContext, protoNode, partitionedScan); PlanProto.PartitionScanSpec partitionScanProto = protoNode.getPartitionScan(); Path [] paths = new Path[partitionScanProto.getPathsCount()]; for (int i = 0; i < partitionScanProto.getPathsCount(); i++) { paths[i] = new Path(partitionScanProto.getPaths(i)); } partitionedScan.setInputPaths(paths); return partitionedScan; }
public String toString() { StringBuilder sb = new StringBuilder("Partitions Scan (table=").append(getTableName()); if (hasAlias()) { sb.append(", alias=").append(alias); } if (hasQual()) { sb.append(", filter=").append(qual); } sb.append(", uri=").append(getTableDesc().getUri()).append(")"); return sb.toString(); }
public void init(ScanNode scanNode, Path[] inputPaths) { tableDesc = scanNode.tableDesc; setInSchema(scanNode.getInSchema()); setOutSchema(scanNode.getOutSchema()); this.qual = scanNode.qual; this.targets = scanNode.targets; setInputPaths(inputPaths); if (scanNode.hasAlias()) { alias = scanNode.alias; } }
@Override public PlanString getPlanString() { PlanString planStr = new PlanString(this).appendTitle(" on " + getTableName()); if (hasAlias()) { planStr.appendTitle(" as ").appendTitle(alias); } if (hasQual()) { planStr.addExplan("filter: ").appendExplain(this.qual.toString()); } if (hasTargets()) { planStr.addExplan("target list: ").appendExplain(StringUtils.join(targets, ", ")); } planStr.addDetail("out schema: ").appendDetail(getOutSchema().toString()); planStr.addDetail("in schema: ").appendDetail(getInSchema().toString()); if (inputPaths != null) { planStr.addExplan("num of filtered paths: ").appendExplain(""+ inputPaths.length); int i = 0; for (Path path : inputPaths) { planStr.addDetail((i++) + ": ").appendDetail(path.toString()); } } return planStr; } }
@Override public LogicalNode visitPartitionedTableScan(Context context, LogicalPlan plan, LogicalPlan.QueryBlock block, PartitionedTableScanNode node, Stack<LogicalNode> stack) throws TajoException { Context newContext = new Context(context); Target [] targets; if (node.hasTargets()) { targets = node.getTargets(); } else { targets = PlannerUtil.schemaToTargets(node.getOutSchema()); } LinkedHashSet<Target> projectedTargets = Sets.newLinkedHashSet(); for (Iterator<Target> it = getFilteredTarget(targets, newContext.requiredSet); it.hasNext();) { Target target = it.next(); newContext.addExpr(target); } for (Iterator<Target> it = context.targetListMgr.getFilteredTargets(newContext.requiredSet); it.hasNext();) { Target target = it.next(); if (LogicalPlanner.checkIfBeEvaluatedAtRelation(block, target.getEvalTree(), node)) { projectedTargets.add(target); newContext.targetListMgr.markAsEvaluated(target); } } node.setTargets(projectedTargets.toArray(new Target[projectedTargets.size()])); LogicalPlanner.verifyProjectedFields(block, node); return node; }
/** * It creates a number of fragments for all partitions. */ public static List<Fragment> getFragmentsFromPartitionedTable(Tablespace tsHandler, ScanNode scan, TableDesc table) throws IOException { Preconditions.checkArgument(tsHandler instanceof FileTablespace, "tsHandler must be FileTablespace"); if (!(scan instanceof PartitionedTableScanNode)) { throw new IllegalArgumentException("scan should be a PartitionedTableScanNode type."); } List<Fragment> fragments = Lists.newArrayList(); PartitionedTableScanNode partitionsScan = (PartitionedTableScanNode) scan; fragments.addAll(((FileTablespace) tsHandler).getSplits( scan.getCanonicalName(), table.getMeta(), table.getSchema(), partitionsScan.getInputPaths())); partitionsScan.setInputPaths(null); return fragments; }
@Override public Object clone() throws CloneNotSupportedException { PartitionedTableScanNode unionScan = (PartitionedTableScanNode) super.clone(); unionScan.tableDesc = (TableDesc) this.tableDesc.clone(); if (hasQual()) { unionScan.qual = (EvalNode) this.qual.clone(); } if (hasTargets()) { unionScan.targets = new ArrayList<>(); for (Target t : targets) { unionScan.targets.add((Target) t.clone()); } } unionScan.inputPaths = inputPaths; return unionScan; }
/** * It creates a number of fragments for all partitions. */ private static List<Fragment> getFragmentsFromPartitionedTable(Tablespace tsHandler, ScanNode scan, TableDesc table, boolean requireSort) throws IOException { Preconditions.checkArgument(tsHandler instanceof FileTablespace, "tsHandler must be FileTablespace"); if (!(scan instanceof PartitionedTableScanNode)) { throw new IllegalArgumentException("scan should be a PartitionedTableScanNode type."); } List<Fragment> fragments = Lists.newArrayList(); PartitionedTableScanNode partitionsScan = (PartitionedTableScanNode) scan; fragments.addAll(((FileTablespace) tsHandler).getSplits( scan.getCanonicalName(), table.getMeta(), table.getSchema(), requireSort, partitionsScan.getInputPaths())); return fragments; }
@Override public Object visitScan(OverridableConf queryContext, LogicalPlan plan, LogicalPlan.QueryBlock block, ScanNode scanNode, Stack<LogicalNode> stack) throws TajoException { TableDesc table = scanNode.getTableDesc(); if (!table.hasPartition()) { return null; } try { Path [] filteredPaths = findFilteredPartitionPaths(queryContext, scanNode); plan.addHistory("PartitionTableRewriter chooses " + filteredPaths.length + " of partitions"); PartitionedTableScanNode rewrittenScanNode = plan.createNode(PartitionedTableScanNode.class); rewrittenScanNode.init(scanNode, filteredPaths); rewrittenScanNode.getTableDesc().getStats().setNumBytes(totalVolume); // if it is topmost node, set it as the rootnode of this block. if (stack.empty() || block.getRoot().equals(scanNode)) { block.setRoot(rewrittenScanNode); } else { PlannerUtil.replaceNode(plan, stack.peek(), scanNode, rewrittenScanNode); } block.registerNode(rewrittenScanNode); } catch (IOException e) { throw new TajoInternalError("Partitioned Table Rewrite Failed: \n" + e.getMessage()); } return null; } }
/** * Clear input paths of {@link PartitionedTableScanNode}. * This is to avoid unnecessary transmission of a lot of partition table paths to workers. * So, this method should be invoked before {@link org.apache.tajo.querymaster.Stage#scheduleFragment(Stage, Fragment)} * unless the scan is broadcasted. * * @param scanNode scan node */ public static void preparePartitionScanPlanForSchedule(ScanNode scanNode) { if (scanNode.getType() == NodeType.PARTITIONS_SCAN) { // TODO: The partition input paths don't have to be kept in a logical node at all. // This should be improved by implementing a specialized fragment for partition tables. PartitionedTableScanNode partitionScan = (PartitionedTableScanNode) scanNode; partitionScan.clearInputPaths(); } } }
@Override public PlanString getPlanString() { PlanString planStr = new PlanString(this).appendTitle(" on " + getTableName()); if (hasAlias()) { planStr.appendTitle(" as ").appendTitle(alias); if (hasQual()) { planStr.addExplan("filter: ").appendExplain(this.qual.toString()); if (hasTargets()) { planStr.addExplan("target list: "); boolean first = true; planStr.addDetail("out schema: ").appendDetail(getOutSchema().toString()); planStr.addDetail("in schema: ").appendDetail(getInSchema().toString());
@Override public LogicalNode visitPartitionedTableScan(Context context, LogicalPlan plan, LogicalPlan.QueryBlock block, PartitionedTableScanNode node, Stack<LogicalNode> stack) throws TajoException { Context newContext = new Context(context); List<Target> targets; if (node.hasTargets()) { targets = node.getTargets(); } else { targets = PlannerUtil.schemaToTargets(node.getOutSchema()); } LinkedHashSet<Target> projectedTargets = Sets.newLinkedHashSet(); for (Iterator<Target> it = getFilteredTarget(targets, newContext.requiredSet); it.hasNext();) { Target target = it.next(); newContext.addExpr(target); } for (Iterator<Target> it = context.targetListMgr.getFilteredTargets(newContext.requiredSet); it.hasNext();) { Target target = it.next(); if (LogicalPlanner.checkIfBeEvaluatedAtRelation(block, target.getEvalTree(), node)) { projectedTargets.add(target); newContext.targetListMgr.markAsEvaluated(target); } } node.setTargets(new ArrayList<>(projectedTargets)); LogicalPlanner.verifyProjectedFields(block, node); return node; }
partitionScanPaths = partitionScan.getInputPaths(); partitionScan.setInputPaths(partitionScanPaths);
@Override public Object clone() throws CloneNotSupportedException { PartitionedTableScanNode unionScan = (PartitionedTableScanNode) super.clone(); unionScan.tableDesc = (TableDesc) this.tableDesc.clone(); if (hasQual()) { unionScan.qual = (EvalNode) this.qual.clone(); } if (hasTargets()) { unionScan.targets = new Target[targets.length]; for (int i = 0; i < targets.length; i++) { unionScan.targets[i] = (Target) targets[i].clone(); } } unionScan.inputPaths = inputPaths; return unionScan; }
@Override public LogicalNode visitPartitionedTableScan(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, PartitionedTableScanNode node, Stack<LogicalNode> stack) throws TajoException { PlanProto.ScanNode.Builder scanBuilder = buildScanNode(node); PlanProto.PartitionScanSpec.Builder partitionScan = PlanProto.PartitionScanSpec.newBuilder(); List<String> pathStrs = new ArrayList<>(); if (node.getInputPaths() != null) { for (Path p : node.getInputPaths()) { pathStrs.add(p.toString()); } partitionScan.addAllPaths(pathStrs); } PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, node); nodeBuilder.setScan(scanBuilder); nodeBuilder.setPartitionScan(partitionScan); context.treeBuilder.addNodes(nodeBuilder); return node; }
@Override public LogicalNode visitPartitionedTableScan(PlanShapeFixerContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, PartitionedTableScanNode node, Stack<LogicalNode> stack) throws TajoException { super.visitPartitionedTableScan(context, plan, block, node, stack); context.childNumbers.push(1); Path[] inputPaths = node.getInputPaths(); Arrays.sort(inputPaths); node.setInputPaths(inputPaths); if (node.hasTargets()) { node.setTargets(sortTargets(node.getTargets())); } if (node.hasQual()) { node.setQual(sortQual(node.getQual())); } return null; }
if (pScanNode.getTableDesc().getStats() == null) { } else { if (pScanNode.getInputPaths() == null || pScanNode.getInputPaths().length == 0) { return 0; } else { return pScanNode.getTableDesc().getStats().getNumBytes();
partitionScanPaths = partitionScan.getInputPaths(); PartitionedTableScanNode partitionScan = (PartitionedTableScanNode)scan; partitionScan.setInputPaths(partitionScanPaths); } else { broadcastFragments.addAll(scanFragments);