@Override public RESULT visitStoreTable(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, StoreTableNode node, Stack<LogicalNode> stack) throws TajoException { stack.push(node); RESULT result = visit(context, plan, block, node.getChild(), stack); stack.pop(); return result; }
public ColPartitionStoreExec(TaskAttemptContext context, StoreTableNode plan, PhysicalExec child) { super(context, plan.getInSchema(), plan.getOutSchema(), child); this.plan = plan; this.outSchema = plan.getTableSchema(); if (this.plan.hasOptions()) { meta = CatalogUtil.newTableMeta(plan.getStorageType(), plan.getOptions()); } else { meta = CatalogUtil.newTableMeta(plan.getStorageType(), context.getConf()); keyNum = this.plan.getPartitionMethod().getExpressionSchema().size(); if (plan.getType() == NodeType.INSERT && keyNum > 0) { Column[] removedPartitionColumns = new Column[this.outSchema.size() - keyNum]; System.arraycopy(this.outSchema.toArray(), 0, removedPartitionColumns, 0, removedPartitionColumns.length); keyNames = new String[keyNum]; for (int i = 0; i < keyNum; i++) { Column column = this.plan.getPartitionMethod().getExpressionSchema().getColumn(i); keyNames[i] = column.getSimpleName(); if (this.plan.getType() == NodeType.INSERT) { InsertNode insertNode = ((InsertNode)plan); int idx = insertNode.getTableSchema().getColumnId(column.getQualifiedName()); keyIds[i] = idx; } else if (this.plan.getType() == NodeType.CREATE_TABLE) { CreateTableNode createTable = (CreateTableNode) plan; int idx = createTable.getLogicalSchema().getColumnId(column.getQualifiedName());
private static PlanProto.StoreTableNodeSpec.Builder buildStoreTableNodeSpec(StoreTableNode node) { PlanProto.StoreTableNodeSpec.Builder storeTableBuilder = PlanProto.StoreTableNodeSpec.newBuilder(); if (node.hasTableName()) { // It will be false if node is for INSERT INTO LOCATION '...' storeTableBuilder.setTableName(node.getTableName()); } if (node.hasUri()) { storeTableBuilder.setUri(node.getUri().toString()); } if (node.hasTableSchema()) { storeTableBuilder.setTableSchema(node.getTableSchema().getProto()); } if (node.hasPartition()) { storeTableBuilder.setPartitionMethod(node.getPartitionMethod().getProto()); } return storeTableBuilder; }
public static TableDesc getOutputTableDesc(LogicalPlan plan) { LogicalNode [] found = findAllNodes(plan.getRootNode().getChild(), NodeType.CREATE_TABLE, NodeType.INSERT); if (found.length == 0) { return new TableDesc(null, plan.getRootNode().getOutSchema(), "TEXT", new KeyValueSet(), null); } else { StoreTableNode storeNode = (StoreTableNode) found[0]; return new TableDesc( storeNode.getTableName(), storeNode.getOutSchema(), storeNode.getStorageType(), storeNode.getOptions(), storeNode.getUri()); } }
private ExecutionBlock buildNoPartitionedStorePlan(GlobalPlanContext context, StoreTableNode currentNode, ExecutionBlock childBlock) throws TajoException { if (hasUnionChild(currentNode)) { // when the below is union return buildShuffleAndStorePlanNoPartitionedTableWithUnion(context, currentNode, childBlock); } else { currentNode.setChild(childBlock.getPlan()); currentNode.setInSchema(childBlock.getPlan().getOutSchema()); childBlock.setPlan(currentNode); return childBlock; } }
private void setShuffleKeysFromPartitionedTableStore(StoreTableNode node, DataChannel channel) { Preconditions.checkState(node.hasTargetTable(), "A target table must be a partitioned table."); PartitionMethodDesc partitionMethod = node.getPartitionMethod(); if (node.getType() == NodeType.INSERT || node.getType() == NodeType.CREATE_TABLE) { Schema tableSchema = null, projectedSchema = null; if (node.getType() == NodeType.INSERT) { tableSchema = ((InsertNode) node).getTableSchema(); projectedSchema = ((InsertNode) node).getProjectedSchema(); } else { tableSchema = node.getOutSchema(); projectedSchema = node.getInSchema(); int i = 0, id = 0; for (Column column : partitionMethod.getExpressionSchema().getRootColumns()) { if (node.getType() == NodeType.INSERT) { id = tableSchema.getColumnId(column.getQualifiedName()); } else {
Schema tableSchema = storeTable.getTableSchema(); sortColumns = getIndexColumns(tableSchema, storeTable.getOptions()); } catch (IOException e) { throw new TajoInternalError(e);
/** * It builds a distributed execution block for CTAS, InsertNode, and StoreTableNode. */ private ExecutionBlock buildStorePlan(GlobalPlanContext context, ExecutionBlock lastBlock, StoreTableNode currentNode) throws TajoException { if(currentNode.hasPartition()) { // if a target table is a partitioned table // Verify supported partition types PartitionMethodDesc partitionMethod = currentNode.getPartitionMethod(); if (partitionMethod.getPartitionType() != CatalogProtos.PartitionType.COLUMN) { throw new NotImplementedException("partition type '" + partitionMethod.getPartitionType().name() + "'"); } if (hasUnionChild(currentNode)) { // if it has union children return buildShuffleAndStorePlanToPartitionedTableWithUnion(context, currentNode, lastBlock); } else { // otherwise return buildShuffleAndStorePlanToPartitionedTable(context, currentNode, lastBlock); } } else { // if result table is not a partitioned table, directly store it return buildNoPartitionedStorePlan(context, currentNode, lastBlock); } }
public void init() throws IOException { super.init(); if (plan.hasOptions()) { meta = CatalogUtil.newTableMeta(plan.getStorageType(), plan.getOptions()); } else { meta = CatalogUtil.newTableMeta(plan.getStorageType(), context.getConf()); } PhysicalPlanUtil.setNullCharIfNecessary(context.getQueryContext(), plan, meta); sumStats = new TableStats(); StoreTableNode storeTableNode = (StoreTableNode) plan; appender = TablespaceManager.get(storeTableNode.getUri()).getAppenderForInsertRow( context.getQueryContext(), context.getTaskId(), meta, storeTableNode.getTableSchema(), context.getOutputPath()); appender.enableStats(); appender.init(); }
/** * It makes a plan to store directly union plans into a non-partitioned table. */ private ExecutionBlock buildShuffleAndStorePlanNoPartitionedTableWithUnion(GlobalPlanContext context, StoreTableNode currentNode, ExecutionBlock childBlock) throws TajoException { for (ExecutionBlock grandChildBlock : context.plan.getChilds(childBlock)) { StoreTableNode copy = PlannerUtil.clone(context.plan.getLogicalPlan(), currentNode); copy.setChild(grandChildBlock.getPlan()); grandChildBlock.setPlan(copy); } return childBlock; }
/** * Create a executor to store a table into HDFS. This is used for CREATE TABLE .. * AS or INSERT (OVERWRITE) INTO statement. */ public PhysicalExec createStorePlan(TaskAttemptContext ctx, StoreTableNode plan, PhysicalExec subOp) throws IOException { if (plan.getPartitionMethod() != null) { switch (plan.getPartitionMethod().getPartitionType()) { case COLUMN: return createColumnPartitionStorePlan(ctx, plan, subOp); default: throw new IllegalStateException(plan.getPartitionMethod().getPartitionType() + " is not supported yet."); } } else { return new StoreTableExec(ctx, plan, subOp); } }
/** * Add partition information to TableStats for storing to CatalogStore. * * @param partition partition name * @throws IOException */ private void addPartition(String partition) throws IOException { PartitionDescProto.Builder builder = PartitionDescProto.newBuilder(); builder.setPartitionName(partition); String[] partitionKeyPairs = partition.split("/"); for (String partitionKeyPair : partitionKeyPairs) { String[] split = partitionKeyPair.split("="); PartitionKeyProto.Builder keyBuilder = PartitionKeyProto.newBuilder(); keyBuilder.setColumnName(split[0]); // Partition path have been escaped to avoid URISyntaxException. But partition value of partition keys table // need to contain unescaped value for comparing filter conditions in select statement. keyBuilder.setPartitionValue(StringUtils.unescapePathName(split[1])); builder.addPartitionKeys(keyBuilder.build()); } if (this.plan.getUri() == null) { // In CTAS, the uri would be null. So, it get the uri from staging directory. int endIndex = storeTablePath.toString().indexOf(FileTablespace.TMP_STAGING_DIR_PREFIX); String outputPath = storeTablePath.toString().substring(0, endIndex); builder.setPath(outputPath + partition); } else { builder.setPath(this.plan.getUri().toString() + "/" + partition); } context.addPartition(builder.build()); }
public static TableDesc getOutputTableDesc(LogicalPlan plan) { LogicalNode [] found = findAllNodes(plan.getRootNode().getChild(), NodeType.CREATE_TABLE, NodeType.INSERT); if (found.length == 0) { return new TableDesc(null, plan.getRootNode().getOutSchema(), "TEXT", new KeyValueSet(), null); } else { StoreTableNode storeNode = (StoreTableNode) found[0]; return new TableDesc( storeNode.getTableName(), storeNode.getOutSchema(), storeNode.getStorageType(), storeNode.getOptions(), storeNode.getUri()); } }
private void setShuffleKeysFromPartitionedTableStore(StoreTableNode node, DataChannel channel) { Preconditions.checkState(node.hasTargetTable(), "A target table must be a partitioned table."); PartitionMethodDesc partitionMethod = node.getPartitionMethod(); if (node.getType() == NodeType.INSERT || node.getType() == NodeType.CREATE_TABLE) { Schema tableSchema = null, projectedSchema = null; if (node.getType() == NodeType.INSERT) { tableSchema = ((InsertNode) node).getTableSchema(); projectedSchema = ((InsertNode) node).getProjectedSchema(); } else { tableSchema = node.getOutSchema(); projectedSchema = node.getInSchema(); int i = 0, id = 0; for (Column column : partitionMethod.getExpressionSchema().getRootColumns()) { if (node.getType() == NodeType.INSERT) { id = tableSchema.getColumnId(column.getQualifiedName()); } else {
private ExecutionBlock buildNoPartitionedStorePlan(GlobalPlanContext context, StoreTableNode currentNode, ExecutionBlock childBlock) throws TajoException { if (hasUnionChild(currentNode)) { // when the below is union return buildShuffleAndStorePlanNoPartitionedTableWithUnion(context, currentNode, childBlock); } else { currentNode.setChild(childBlock.getPlan()); currentNode.setInSchema(childBlock.getPlan().getOutSchema()); childBlock.setPlan(currentNode); return childBlock; } }
Schema tableSchema = storeTable.getTableSchema(); sortColumns = getIndexColumns(tableSchema, storeTable.getOptions()); } catch (IOException e) { throw new TajoInternalError(e);
/** * It builds a distributed execution block for CTAS, InsertNode, and StoreTableNode. */ private ExecutionBlock buildStorePlan(GlobalPlanContext context, ExecutionBlock lastBlock, StoreTableNode currentNode) throws TajoException { if(currentNode.hasPartition()) { // if a target table is a partitioned table // Verify supported partition types PartitionMethodDesc partitionMethod = currentNode.getPartitionMethod(); if (partitionMethod.getPartitionType() != CatalogProtos.PartitionType.COLUMN) { throw new NotImplementedException("partition type '" + partitionMethod.getPartitionType().name() + "'"); } if (hasUnionChild(currentNode)) { // if it has union children return buildShuffleAndStorePlanToPartitionedTableWithUnion(context, currentNode, lastBlock); } else { // otherwise return buildShuffleAndStorePlanToPartitionedTable(context, currentNode, lastBlock); } } else { // if result table is not a partitioned table, directly store it return buildNoPartitionedStorePlan(context, currentNode, lastBlock); } }
public void init() throws IOException { super.init(); if (plan.hasOptions()) { meta = CatalogUtil.newTableMeta(plan.getStorageType(), plan.getOptions()); } else { meta = CatalogUtil.newTableMeta(plan.getStorageType(), context.getConf()); } PhysicalPlanUtil.setNullCharIfNecessary(context.getQueryContext(), plan, meta); sumStats = new TableStats(); StoreTableNode storeTableNode = (StoreTableNode) plan; appender = TablespaceManager.get(storeTableNode.getUri()).getAppenderForInsertRow( context.getQueryContext(), context.getTaskId(), meta, storeTableNode.getTableSchema(), context.getOutputPath()); appender.enableStats(); appender.init(); }
/** * It makes a plan to store directly union plans into a non-partitioned table. */ private ExecutionBlock buildShuffleAndStorePlanNoPartitionedTableWithUnion(GlobalPlanContext context, StoreTableNode currentNode, ExecutionBlock childBlock) throws TajoException { for (ExecutionBlock grandChildBlock : context.plan.getChilds(childBlock)) { StoreTableNode copy = PlannerUtil.clone(context.plan.getLogicalPlan(), currentNode); copy.setChild(grandChildBlock.getPlan()); grandChildBlock.setPlan(copy); } return childBlock; }
/** * Create a executor to store a table into HDFS. This is used for CREATE TABLE .. * AS or INSERT (OVERWRITE) INTO statement. */ public PhysicalExec createStorePlan(TaskAttemptContext ctx, StoreTableNode plan, PhysicalExec subOp) throws IOException { if (plan.getPartitionMethod() != null) { switch (plan.getPartitionMethod().getPartitionType()) { case COLUMN: return createColumnPartitionStorePlan(ctx, plan, subOp); default: throw new IllegalStateException(plan.getPartitionMethod().getPartitionType() + " is not supported yet."); } } else { return new StoreTableExec(ctx, plan, subOp); } }