/** * Add partition information to TableStats for storing to CatalogStore. * * @param partition partition name * @throws IOException */ private void addPartition(String partition) throws IOException { PartitionDescProto.Builder builder = PartitionDescProto.newBuilder(); builder.setPartitionName(partition); String[] partitionKeyPairs = partition.split("/"); for (String partitionKeyPair : partitionKeyPairs) { String[] split = partitionKeyPair.split("="); PartitionKeyProto.Builder keyBuilder = PartitionKeyProto.newBuilder(); keyBuilder.setColumnName(split[0]); // Partition path have been escaped to avoid URISyntaxException. But partition value of partition keys table // need to contain unescaped value for comparing filter conditions in select statement. keyBuilder.setPartitionValue(StringUtils.unescapePathName(split[1])); builder.addPartitionKeys(keyBuilder.build()); } if (this.plan.getUri() == null) { // In CTAS, the uri would be null. So, it get the uri from staging directory. int endIndex = storeTablePath.toString().indexOf(FileTablespace.TMP_STAGING_DIR_PREFIX); String outputPath = storeTablePath.toString().substring(0, endIndex); builder.setPath(outputPath + partition); } else { builder.setPath(this.plan.getUri().toString() + "/" + partition); } context.addPartition(builder.build()); }
/** * Add partition information to TableStats for storing to CatalogStore. * * @param partition partition name * @throws IOException */ private void addPartition(String partition) throws IOException { PartitionDescProto.Builder builder = PartitionDescProto.newBuilder(); builder.setPartitionName(partition); String[] partitionKeyPairs = partition.split("/"); for(int i = 0; i < partitionKeyPairs.length; i++) { String partitionKeyPair = partitionKeyPairs[i]; String[] split = partitionKeyPair.split("="); PartitionKeyProto.Builder keyBuilder = PartitionKeyProto.newBuilder(); keyBuilder.setColumnName(split[0]); // Partition path have been escaped to avoid URISyntaxException. But partition value of partition keys table // need to contain unescaped value for comparing filter conditions in select statement. keyBuilder.setPartitionValue(StringUtils.unescapePathName(split[1])); builder.addPartitionKeys(keyBuilder.build()); } if (this.plan.getUri() == null) { // In CTAS, the uri would be null. So, it get the uri from staging directory. int endIndex = storeTablePath.toString().indexOf(FileTablespace.TMP_STAGING_DIR_PREFIX); String outputPath = storeTablePath.toString().substring(0, endIndex); builder.setPath(outputPath + partition); } else { builder.setPath(this.plan.getUri().toString() + "/" + partition); } context.addPartition(builder.build()); }
private static PlanProto.StoreTableNodeSpec.Builder buildStoreTableNodeSpec(StoreTableNode node) { PlanProto.StoreTableNodeSpec.Builder storeTableBuilder = PlanProto.StoreTableNodeSpec.newBuilder(); if (node.hasTableName()) { // It will be false if node is for INSERT INTO LOCATION '...' storeTableBuilder.setTableName(node.getTableName()); } if (node.hasUri()) { storeTableBuilder.setUri(node.getUri().toString()); } if (node.hasTableSchema()) { storeTableBuilder.setTableSchema(node.getTableSchema().getProto()); } if (node.hasPartition()) { storeTableBuilder.setPartitionMethod(node.getPartitionMethod().getProto()); } return storeTableBuilder; }
private static PlanProto.StoreTableNodeSpec.Builder buildStoreTableNodeSpec(StoreTableNode node) { PlanProto.StoreTableNodeSpec.Builder storeTableBuilder = PlanProto.StoreTableNodeSpec.newBuilder(); if (node.hasTableName()) { // It will be false if node is for INSERT INTO LOCATION '...' storeTableBuilder.setTableName(node.getTableName()); } if (node.hasUri()) { storeTableBuilder.setUri(node.getUri().toString()); } if (node.hasTableSchema()) { storeTableBuilder.setTableSchema(node.getTableSchema().getProto()); } if (node.hasPartition()) { storeTableBuilder.setPartitionMethod(node.getPartitionMethod().getProto()); } return storeTableBuilder; }
public static TableDesc getOutputTableDesc(LogicalPlan plan) { LogicalNode [] found = findAllNodes(plan.getRootNode().getChild(), NodeType.CREATE_TABLE, NodeType.INSERT); if (found.length == 0) { return new TableDesc(null, plan.getRootNode().getOutSchema(), "TEXT", new KeyValueSet(), null); } else { StoreTableNode storeNode = (StoreTableNode) found[0]; return new TableDesc( storeNode.getTableName(), storeNode.getOutSchema(), storeNode.getStorageType(), storeNode.getOptions(), storeNode.getUri()); } }
public static TableDesc getOutputTableDesc(LogicalPlan plan) { LogicalNode [] found = findAllNodes(plan.getRootNode().getChild(), NodeType.CREATE_TABLE, NodeType.INSERT); if (found.length == 0) { return new TableDesc(null, plan.getRootNode().getOutSchema(), "TEXT", new KeyValueSet(), null); } else { StoreTableNode storeNode = (StoreTableNode) found[0]; return new TableDesc( storeNode.getTableName(), storeNode.getOutSchema(), storeNode.getStorageType(), storeNode.getOptions(), storeNode.getUri()); } }
public void init() throws IOException { super.init(); if (plan.hasOptions()) { meta = CatalogUtil.newTableMeta(plan.getStorageType(), plan.getOptions()); } else { meta = CatalogUtil.newTableMeta(plan.getStorageType(), context.getConf()); } PhysicalPlanUtil.setNullCharIfNecessary(context.getQueryContext(), plan, meta); sumStats = new TableStats(); StoreTableNode storeTableNode = (StoreTableNode) plan; appender = TablespaceManager.get(storeTableNode.getUri()).getAppenderForInsertRow( context.getQueryContext(), context.getTaskId(), meta, storeTableNode.getTableSchema(), context.getOutputPath()); appender.enableStats(); appender.init(); }
public void init() throws IOException { super.init(); if (plan.hasOptions()) { meta = CatalogUtil.newTableMeta(plan.getStorageType(), plan.getOptions()); } else { meta = CatalogUtil.newTableMeta(plan.getStorageType(), context.getConf()); } PhysicalPlanUtil.setNullCharIfNecessary(context.getQueryContext(), plan, meta); sumStats = new TableStats(); StoreTableNode storeTableNode = (StoreTableNode) plan; appender = TablespaceManager.get(storeTableNode.getUri()).getAppenderForInsertRow( context.getQueryContext(), context.getTaskId(), meta, storeTableNode.getTableSchema(), context.getOutputPath()); appender.enableStats(); appender.init(); }