LogicalNode child = insertNode.getChild(); insertNode.setInSchema(childSchema); insertNode.setOutSchema(childSchema); insertNode.setTableSchema(childSchema); targetUri = URI.create(context.getQueryContext().get(QueryVars.DEFAULT_SPACE_ROOT_URI) + "/" + targetUri); insertNode.setUri(targetUri); insertNode.setDataFormat(CatalogUtil.getBackwardCompitableDataFormat(expr.getDataFormat())); insertNode.setOptions(options);
@Override public PlanString getPlanString() { PlanString planString = new PlanString(this); planString.appendTitle(" INTO "); if (hasTargetTable()) { planString.appendTitle(getTableName()); if (hasTargetSchema()) { planString.addExplan(getTargetSchema().toString()); } } else { planString.addExplan("LOCATION " + uri); } return planString; } }
@Override public void hook(QueryContext queryContext, LogicalPlan plan) throws Exception { queryContext.setInsert(); InsertNode insertNode = plan.getRootBlock().getNode(NodeType.INSERT); // Set QueryContext settings, such as output table name and output path. // It also remove data files if overwrite is true. if (insertNode.hasTargetTable()) { // INSERT INTO [TB_NAME] queryContext.setOutputTable(insertNode.getTableName()); if (insertNode.hasPartition()) { queryContext.setPartitionMethod(insertNode.getPartitionMethod()); } } else { // INSERT INTO LOCATION ... // When INSERT INTO LOCATION, must not set output table. queryContext.setFileOutput(); } // Set the final output table uri queryContext.setOutputPath(insertNode.getUri()); if (insertNode.isOverwrite()) { queryContext.setOutputOverwrite(); } } }
private static TableDesc createTableDesc(InsertNode insertNode) { TableMeta meta = new TableMeta(insertNode.getStorageType(), insertNode.getOptions()); TableDesc tableDescTobeCreated = new TableDesc( insertNode.getTableName(), insertNode.getTableSchema(), meta, insertNode.getUri() != null ? insertNode.getUri() : null); if (insertNode.hasPartition()) { tableDescTobeCreated.setPartitionMethod(insertNode.getPartitionMethod()); } return tableDescTobeCreated; }
public String toString() { StringBuilder sb = new StringBuilder("Insert (overwrite=").append(overwrite); if (hasTargetTable()) { sb.append(",table=").append(tableName); } if (hasUri()) { sb.append(", location=").append(uri); } sb.append(")"); return sb.toString(); }
InsertNode insertNode, SubmitQueryResponse.Builder responseBuilder) { try { String nodeUniqName = insertNode.getTableName() == null ? new Path(insertNode.getUri()).getName() : insertNode.getTableName(); String queryId = nodeUniqName + "_" + System.currentTimeMillis(); URI finalOutputUri = insertNode.getUri(); Tablespace space = TablespaceManager.get(finalOutputUri); TableMeta tableMeta = new TableMeta(insertNode.getStorageType(), insertNode.getOptions()); tableMeta.putOption(StorageConstants.INSERT_DIRECTLY, Boolean.TRUE.toString()); taskAttemptContext.setOutputPath(new Path(finalOutputUri)); EvalExprExec evalExprExec = new EvalExprExec(taskAttemptContext, (EvalExprNode) insertNode.getChild()); InsertRowsExec exec = new InsertRowsExec(taskAttemptContext, insertNode, evalExprExec); stats.setNumRows(taskAttemptContext.getResultStats().getNumRows()); if (insertNode.hasTargetTable()) { CatalogProtos.UpdateTableStatsProto.Builder builder = CatalogProtos.UpdateTableStatsProto.newBuilder(); builder.setTableName(insertNode.getTableName()); builder.setStats(stats.getProto()); insertNode.getTableName(), insertNode.getTargetSchema(), tableMeta, finalOutputUri);
@Override public LogicalNode visitInsert(SerializeContext context, LogicalPlan plan, LogicalPlan.QueryBlock block, InsertNode node, Stack<LogicalNode> stack) throws TajoException { super.visitInsert(context, plan, block, node, stack); int [] childIds = registerGetChildIds(context, node); PlanProto.PersistentStoreNode.Builder persistentStoreBuilder = buildPersistentStoreBuilder(node, childIds); PlanProto.StoreTableNodeSpec.Builder storeTableBuilder = buildStoreTableNodeSpec(node); PlanProto.InsertNodeSpec.Builder insertNodeSpec = PlanProto.InsertNodeSpec.newBuilder(); insertNodeSpec.setOverwrite(node.isOverwrite()); if (node.hasProjectedSchema()) { insertNodeSpec.setProjectedSchema(node.getProjectedSchema().getProto()); } if (node.hasTargetSchema()) { insertNodeSpec.setTargetSchema(node.getTargetSchema().getProto()); } PlanProto.LogicalNode.Builder nodeBuilder = createNodeBuilder(context, node); nodeBuilder.setPersistentStore(persistentStoreBuilder); nodeBuilder.setStoreTable(storeTableBuilder); nodeBuilder.setInsert(insertNodeSpec); context.treeBuilder.addNodes(nodeBuilder); return node; }
private void buildProjectedInsert(PlanContext context, InsertNode insertNode) { Schema tableSchema = insertNode.getTableSchema(); Schema targetColumns = insertNode.getTargetSchema(); LogicalNode child = insertNode.getChild(); Projectable projectionNode = insertNode.getChild(); insertNode.setInSchema(projectionNode.getOutSchema()); insertNode.setOutSchema(projectionNode.getOutSchema()); insertNode.setProjectedSchema(PlannerUtil.targetToSchema(targets)); } else { throw new RuntimeException("Wrong child node type: " + child.getType() + " for insert");
@Test public final void testInsertInto0() throws TajoException { QueryContext qc = createQueryContext(); Expr expr = sqlAnalyzer.parse(insertStatements[0]); LogicalPlan plan = planner.createPlan(qc, expr); assertEquals(1, plan.getQueryBlocks().size()); InsertNode insertNode = getInsertNode(plan); assertFalse(insertNode.isOverwrite()); assertTrue(insertNode.hasTargetTable()); assertEquals(IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "score"), insertNode.getTableName()); }
@Override public RESULT visitInsert(CONTEXT context, LogicalPlan plan, LogicalPlan.QueryBlock block, InsertNode node, Stack<LogicalNode> stack) throws TajoException { stack.push(node); RESULT result = visit(context, plan, block, node.getChild(), stack); stack.pop(); return result; }
insertNode.setTargetTable(desc); if (expr.getTargetColumns().length > insertNode.getChild().getOutSchema().size()) { throw makeSyntaxError("Target columns and projected columns are mismatched to each other"); insertNode.setTargetSchema(targetColumns); insertNode.setOutSchema(targetColumns); buildProjectedInsert(context, insertNode); Schema projectedSchema = insertNode.getChild().getOutSchema(); targetColumns.addColumn(tableSchema.getColumn(i)); insertNode.setTargetSchema(targetColumns); buildProjectedInsert(context, insertNode); insertNode.setPartitionMethod(desc.getPartitionMethod());
if (insertNode.hasTargetTable()) { String tableName = insertNode.getTableName(); finalTable = catalog.getTableDesc(tableName); } else { if (insertNode.hasTargetTable()) { UpdateTableStatsProto.Builder builder = UpdateTableStatsProto.newBuilder(); builder.setTableName(finalTable.getName());
throws IOException { EvalExprExec evalExprExec = new EvalExprExec(taskAttemptContext, (EvalExprNode) insertNode.getChild()); InsertRowsExec exec = new InsertRowsExec(taskAttemptContext, insertNode, evalExprExec); if (insertNode.isOverwrite()) { // INSERT OVERWRITE INTO
@Test public final void testInsertInto1() throws TajoException { QueryContext qc = createQueryContext(); Expr expr = sqlAnalyzer.parse(insertStatements[1]); LogicalPlan plan = planner.createPlan(qc, expr); assertEquals(1, plan.getQueryBlocks().size()); InsertNode insertNode = getInsertNode(plan); assertFalse(insertNode.isOverwrite()); assertEquals(IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "score"), insertNode.getTableName()); }
for (int i = 0; i < partitionKeyColumns.length; i++) { for (Column column : partitionKeyColumns) { int id = insertNode.getTableSchema().getColumnId(column.getQualifiedName()); sortSpecs[i++] = new SortSpec(insertNode.getProjectedSchema().getColumn(id), true, false);
@Test public final void testInsertInto3() throws TajoException { QueryContext qc = createQueryContext(); Expr expr = sqlAnalyzer.parse(insertStatements[3]); LogicalPlan plan = planner.createPlan(qc, expr); assertEquals(1, plan.getQueryBlocks().size()); InsertNode insertNode = getInsertNode(plan); assertFalse(insertNode.isOverwrite()); assertTrue(insertNode.hasUri()); }
public static TableDesc getTableDesc(CatalogService catalog, LogicalNode node) throws UndefinedTableException { if (node.getType() == NodeType.ROOT) { node = ((LogicalRootNode)node).getChild(); } if (node.getType() == NodeType.CREATE_TABLE) { return createTableDesc((CreateTableNode)node); } String tableName = null; InsertNode insertNode = null; if (node.getType() == NodeType.INSERT) { insertNode = (InsertNode)node; tableName = insertNode.getTableName(); } else { return null; } if (tableName != null) { String[] tableTokens = tableName.split("\\."); if (tableTokens.length >= 2) { if (catalog.existsTable(tableTokens[0], tableTokens[1])) { return catalog.getTableDesc(tableTokens[0], tableTokens[1]); } } } else { if (insertNode.getUri() != null) { //insert ... location return createTableDesc(insertNode); } } return null; }
private ProjectionNode makeProjectionForInsertUnion(PlanContext context, InsertNode insertNode) { LogicalNode child = insertNode.getChild(); // add (projection - subquery) to RootBlock and create new QueryBlock for UnionNode TableSubQueryNode subQueryNode = context.plan.createNode(TableSubQueryNode.class); subQueryNode.init(context.queryBlock.getName(), child); subQueryNode.setTargets(PlannerUtil.schemaToTargets(subQueryNode.getOutSchema())); ProjectionNode projectionNode = context.plan.createNode(ProjectionNode.class); projectionNode.setChild(subQueryNode); projectionNode.setInSchema(subQueryNode.getInSchema()); projectionNode.setTargets(subQueryNode.getTargets()); context.queryBlock.registerNode(projectionNode); context.queryBlock.registerNode(subQueryNode); // add child QueryBlock to the UnionNode's QueryBlock UnionNode unionNode = (UnionNode)child; context.queryBlock.unregisterNode(unionNode); QueryBlock unionBlock = context.plan.newQueryBlock(); unionBlock.registerNode(unionNode); unionBlock.setRoot(unionNode); QueryBlock leftBlock = context.plan.getBlock(unionNode.getLeftChild()); QueryBlock rightBlock = context.plan.getBlock(unionNode.getRightChild()); context.plan.disconnectBlocks(leftBlock, context.queryBlock); context.plan.disconnectBlocks(rightBlock, context.queryBlock); context.plan.connectBlocks(unionBlock, context.queryBlock, BlockType.TableSubQuery); context.plan.connectBlocks(leftBlock, unionBlock, BlockType.TableSubQuery); context.plan.connectBlocks(rightBlock, unionBlock, BlockType.TableSubQuery); // set InsertNode's child with ProjectionNode which is created. insertNode.setChild(projectionNode); return projectionNode; }
public static String getStoreTableName(LogicalPlan plan) { LogicalRootNode rootNode = plan.getRootBlock().getRoot(); NodeType nodeType = rootNode.getChild().getType(); if (nodeType == NodeType.CREATE_TABLE) { return ((CreateTableNode)rootNode.getChild()).getTableName(); } else if (nodeType == NodeType.INSERT) { return ((InsertNode)rootNode.getChild()).getTableName(); } else { return null; } }
public void openNewFile(int suffixId) throws IOException { Schema appenderSchema = (plan instanceof InsertNode) ? ((InsertNode) plan).getTableSchema() : outSchema; if (PlannerUtil.isFileStorageType(meta.getDataFormat())) { String prevFile = null; lastFileName = context.getOutputPath(); if (suffixId > 0) { prevFile = lastFileName.toString(); lastFileName = new Path(lastFileName + "_" + suffixId); } FileTablespace space = TablespaceManager.get(lastFileName.toUri()); appender = space.getAppender(meta, appenderSchema, lastFileName); if (suffixId > 0) { LOG.info(prevFile + " exceeds " + SessionVars.MAX_OUTPUT_FILE_SIZE.keyname() + " (" + maxPerFileSize + " MB), " + "The remain output will be written into " + lastFileName.toString()); } } else { Path stagingDir = context.getQueryContext().getStagingDir(); appender = TablespaceManager.get(stagingDir.toUri()).getAppender( context.getQueryContext(), context.getTaskId(), meta, appenderSchema, stagingDir); } appender.enableStats(); appender.init(); }