/** * Create a executor to store a table into HDFS. This is used for CREATE TABLE .. * AS or INSERT (OVERWRITE) INTO statement. */ public PhysicalExec createStorePlan(TaskAttemptContext ctx, StoreTableNode plan, PhysicalExec subOp) throws IOException { if (plan.getPartitionMethod() != null) { switch (plan.getPartitionMethod().getPartitionType()) { case COLUMN: return createColumnPartitionStorePlan(ctx, plan, subOp); default: throw new IllegalStateException(plan.getPartitionMethod().getPartitionType() + " is not supported yet."); } } else { return new StoreTableExec(ctx, plan, subOp); } }
/** * Create a executor to store a table into HDFS. This is used for CREATE TABLE .. * AS or INSERT (OVERWRITE) INTO statement. */ public PhysicalExec createStorePlan(TaskAttemptContext ctx, StoreTableNode plan, PhysicalExec subOp) throws IOException { if (plan.getPartitionMethod() != null) { switch (plan.getPartitionMethod().getPartitionType()) { case COLUMN: return createColumnPartitionStorePlan(ctx, plan, subOp); default: throw new IllegalStateException(plan.getPartitionMethod().getPartitionType() + " is not supported yet."); } } else { return new StoreTableExec(ctx, plan, subOp); } }
private static PlanProto.StoreTableNodeSpec.Builder buildStoreTableNodeSpec(StoreTableNode node) { PlanProto.StoreTableNodeSpec.Builder storeTableBuilder = PlanProto.StoreTableNodeSpec.newBuilder(); if (node.hasTableName()) { // It will be false if node is for INSERT INTO LOCATION '...' storeTableBuilder.setTableName(node.getTableName()); } if (node.hasUri()) { storeTableBuilder.setUri(node.getUri().toString()); } if (node.hasTableSchema()) { storeTableBuilder.setTableSchema(node.getTableSchema().getProto()); } if (node.hasPartition()) { storeTableBuilder.setPartitionMethod(node.getPartitionMethod().getProto()); } return storeTableBuilder; }
private static PlanProto.StoreTableNodeSpec.Builder buildStoreTableNodeSpec(StoreTableNode node) { PlanProto.StoreTableNodeSpec.Builder storeTableBuilder = PlanProto.StoreTableNodeSpec.newBuilder(); if (node.hasTableName()) { // It will be false if node is for INSERT INTO LOCATION '...' storeTableBuilder.setTableName(node.getTableName()); } if (node.hasUri()) { storeTableBuilder.setUri(node.getUri().toString()); } if (node.hasTableSchema()) { storeTableBuilder.setTableSchema(node.getTableSchema().getProto()); } if (node.hasPartition()) { storeTableBuilder.setPartitionMethod(node.getPartitionMethod().getProto()); } return storeTableBuilder; }
/** * It builds a distributed execution block for CTAS, InsertNode, and StoreTableNode. */ private ExecutionBlock buildStorePlan(GlobalPlanContext context, ExecutionBlock lastBlock, StoreTableNode currentNode) throws TajoException { if(currentNode.hasPartition()) { // if a target table is a partitioned table // Verify supported partition types PartitionMethodDesc partitionMethod = currentNode.getPartitionMethod(); if (partitionMethod.getPartitionType() != CatalogProtos.PartitionType.COLUMN) { throw new NotImplementedException("partition type '" + partitionMethod.getPartitionType().name() + "'"); } if (hasUnionChild(currentNode)) { // if it has union children return buildShuffleAndStorePlanToPartitionedTableWithUnion(context, currentNode, lastBlock); } else { // otherwise return buildShuffleAndStorePlanToPartitionedTable(context, currentNode, lastBlock); } } else { // if result table is not a partitioned table, directly store it return buildNoPartitionedStorePlan(context, currentNode, lastBlock); } }
/** * It builds a distributed execution block for CTAS, InsertNode, and StoreTableNode. */ private ExecutionBlock buildStorePlan(GlobalPlanContext context, ExecutionBlock lastBlock, StoreTableNode currentNode) throws TajoException { if(currentNode.hasPartition()) { // if a target table is a partitioned table // Verify supported partition types PartitionMethodDesc partitionMethod = currentNode.getPartitionMethod(); if (partitionMethod.getPartitionType() != CatalogProtos.PartitionType.COLUMN) { throw new NotImplementedException("partition type '" + partitionMethod.getPartitionType().name() + "'"); } if (hasUnionChild(currentNode)) { // if it has union children return buildShuffleAndStorePlanToPartitionedTableWithUnion(context, currentNode, lastBlock); } else { // otherwise return buildShuffleAndStorePlanToPartitionedTable(context, currentNode, lastBlock); } } else { // if result table is not a partitioned table, directly store it return buildNoPartitionedStorePlan(context, currentNode, lastBlock); } }
PhysicalExec child) throws IOException { Column[] partitionKeyColumns = storeTableNode.getPartitionMethod().getExpressionSchema().toArray(); SortSpec[] sortSpecs = new SortSpec[partitionKeyColumns.length];
PhysicalExec child) throws IOException { Column[] partitionKeyColumns = storeTableNode.getPartitionMethod().getExpressionSchema().toArray(); SortSpec[] sortSpecs = new SortSpec[partitionKeyColumns.length];
private void setShuffleKeysFromPartitionedTableStore(StoreTableNode node, DataChannel channel) { Preconditions.checkState(node.hasTargetTable(), "A target table must be a partitioned table."); PartitionMethodDesc partitionMethod = node.getPartitionMethod();
private void setShuffleKeysFromPartitionedTableStore(StoreTableNode node, DataChannel channel) { Preconditions.checkState(node.hasTargetTable(), "A target table must be a partitioned table."); PartitionMethodDesc partitionMethod = node.getPartitionMethod();
keyNum = this.plan.getPartitionMethod().getExpressionSchema().size(); keyNames = new String[keyNum]; for (int i = 0; i < keyNum; i++) { Column column = this.plan.getPartitionMethod().getExpressionSchema().getColumn(i); keyNames[i] = column.getSimpleName();
keyNum = this.plan.getPartitionMethod().getExpressionSchema().size(); keyNames = new String[keyNum]; for (int i = 0; i < keyNum; i++) { Column column = this.plan.getPartitionMethod().getExpressionSchema().getColumn(i); keyNames[i] = column.getSimpleName();
} else if (node instanceof StoreTableNode && ((StoreTableNode)node).hasPartition() && ((StoreTableNode)node).getPartitionMethod().getPartitionType() == PartitionType.COLUMN) { type = EnforceType.COLUMN_PARTITION; } else {
} else if (node instanceof StoreTableNode && ((StoreTableNode)node).hasPartition() && ((StoreTableNode)node).getPartitionMethod().getPartitionType() == PartitionType.COLUMN) { type = EnforceType.COLUMN_PARTITION; } else {