public void setTargetTable(TableDesc desc) { setTableName(desc.getName()); if (desc.hasPartition()) { tableSchema = desc.getLogicalSchema(); } else { tableSchema = desc.getSchema(); } if (desc.getUri() != null) { setUri(desc.getUri()); } setOptions(desc.getMeta().getPropertySet()); setDataFormat(desc.getMeta().getDataFormat()); if (desc.hasPartition()) { this.setPartitionMethod(desc.getPartitionMethod()); } }
public static ResultSet createResultSet(TajoClient client, QueryId queryId, ClientProtos.GetQueryResultResponse response, int fetchRows) throws IOException { TableDesc desc = new TableDesc(response.getTableDesc()); return new FetchResultSet(client, desc.getLogicalSchema(), queryId, fetchRows); }
@Override public void createTable(TableDesc tableDesc, boolean ifNotExists) throws TajoException, IOException { createTable(tableDesc.getUri(), tableDesc.getMeta(), tableDesc.getSchema(), tableDesc.isExternal(), ifNotExists); TableStats stats = new TableStats(); stats.setNumRows(TajoConstants.UNKNOWN_ROW_NUMBER); tableDesc.setStats(stats); }
public LogicalNode handleCreateTableLike(PlanContext context, CreateTable expr, CreateTableNode createTableNode) throws TajoException { String parentTableName = expr.getLikeParentTableName(); if (CatalogUtil.isFQTableName(parentTableName) == false) { parentTableName = CatalogUtil.buildFQName(context.queryContext.get(SessionVars.CURRENT_DATABASE), parentTableName); } TableDesc baseTable = catalog.getTableDesc(parentTableName); if(baseTable == null) { throw new UndefinedTableException(parentTableName); } PartitionMethodDesc partitionDesc = baseTable.getPartitionMethod(); createTableNode.setTableSchema(baseTable.getSchema()); createTableNode.setPartitionMethod(partitionDesc); createTableNode.setDataFormat(CatalogUtil.getBackwardCompitableDataFormat(baseTable.getMeta().getDataFormat())); createTableNode.setOptions(baseTable.getMeta().getOptions()); createTableNode.setExternal(baseTable.isExternal()); if(baseTable.isExternal()) { createTableNode.setUri(baseTable.getUri()); } return createTableNode; }
@Override public void createTable(TableDesc tableDesc, boolean ifNotExists) throws IOException { if (!tableDesc.isExternal()) { String [] splitted = IdentifierUtil.splitFQTableName(tableDesc.getName()); String databaseName = splitted[0]; String simpleTableName = splitted[1]; tableDesc.setUri(tablePath.toUri()); } else { Preconditions.checkState(tableDesc.getUri() != null, "ERROR: LOCATION must be given."); Path path = new Path(tableDesc.getUri()); if (tableDesc.isExternal()) { if (!fs.exists(path)) { LOG.error(path.toUri() + " does not exist"); if (tableDesc.isExternal()) { // if it is an external table, there is no way to know the exact row number without processing. stats.setNumRows(TajoConstants.UNKNOWN_ROW_NUMBER); tableDesc.setStats(stats);
public static String buildDDLForBaseTable(TableDesc desc) { StringBuilder sb = new StringBuilder(); sb.append("--\n") .append("-- Name: ").append(CatalogUtil.denormalizeIdentifier(desc.getName())).append("; Type: TABLE;") .append(" Storage: ").append(desc.getMeta().getDataFormat()); sb.append("\n--\n"); sb.append("CREATE TABLE ").append(CatalogUtil.denormalizeIdentifier(desc.getName())); buildSchema(sb, desc.getSchema()); buildUsingClause(sb, desc.getMeta()); buildWithClause(sb, desc.getMeta()); if (desc.hasPartition()) { buildPartitionClause(sb, desc); } sb.append(";"); return sb.toString(); }
} else { String tableName = query.getId().toString(); finalTable = new TableDesc(tableName, lastStage.getOutSchema(), meta, finalOutputDir.toUri()); finalTable.setStats(aggregated); } else { long volume = getTableVolume(query.systemConf, finalOutputDir); stats.setNumBytes(volume); finalTable.setStats(stats); builder.setTableName(finalTable.getName()); builder.setStats(finalTable.getStats().getProto());
if (tableDesc.isExternal()) { throw new TajoRuntimeException( new UnsupportedException("table truncation on an external table '" + eachTableName + "'")); Tablespace space = TablespaceManager.get(tableDesc.getUri()); Path path = new Path(eachTable.getUri()); LOG.info("Truncate table: " + eachTable.getName() + ", delete all data files in " + path); FileSystem fs = path.getFileSystem(context.getConf());
@Override public void purgeTable(TableDesc tableDesc) throws IOException { try { Path path = new Path(tableDesc.getUri()); FileSystem fs = path.getFileSystem(conf); LOG.info("Delete table data dir: " + path); fs.delete(path, true); } catch (IOException e) { throw new InternalError(e.getMessage()); } }
@Override public void execute(QueryMaster.QueryMasterContext context, QueryContext queryContext, Query query, ExecutionBlockId finalExecBlockId, Path finalOutputDir) throws Exception { CatalogService catalog = context.getWorkerContext().getCatalog(); Stage lastStage = query.getStage(finalExecBlockId); TableStats stats = lastStage.getResultStats(); CreateTableNode createTableNode = (CreateTableNode) lastStage.getBlock().getPlan(); TableMeta meta = new TableMeta(createTableNode.getStorageType(), createTableNode.getOptions()); TableDesc tableDescTobeCreated = new TableDesc( createTableNode.getTableName(), createTableNode.getTableSchema(), meta, finalOutputDir.toUri()); tableDescTobeCreated.setExternal(createTableNode.isExternal()); if (createTableNode.hasPartition()) { tableDescTobeCreated.setPartitionMethod(createTableNode.getPartitionMethod()); } if (query.hasUnionPlan()) { TableStats aggregated = query.aggregateTableStatsOfTerminalBlock(); tableDescTobeCreated.setStats(aggregated); } else { stats.setNumBytes(getTableVolume(query.systemConf, finalOutputDir)); tableDescTobeCreated.setStats(stats); } query.setResultDesc(tableDescTobeCreated); catalog.createTable(tableDescTobeCreated); } }
@Test(timeout = 1000) public void testGetTable() throws UndefinedTableException { for (String tableName: PgSQLTestServer.TPCH_TABLES) { TableDesc retrieved = client.getTableDesc(PgSQLTestServer.DATABASE_NAME + "." + tableName); assertEquals(PgSQLTestServer.DATABASE_NAME + "." + tableName, retrieved.getName()); assertEquals(jdbcUrl + "&table=" + tableName, retrieved.getUri().toASCIIString()); } } }
private void prepareForCreateTableOrInsert(CatalogService catalog, LogicalPlan plan) throws TajoException, IOException { LogicalRootNode rootNode = plan.getRootBlock().getRoot(); TableDesc tableDesc = PlannerUtil.getTableDesc(catalog, plan.getRootBlock().getRoot()); if (tableDesc != null) { Tablespace space = TablespaceManager.get(tableDesc.getUri()); FormatProperty formatProperty = space.getFormatProperty(tableDesc.getMeta()); if (!formatProperty.isInsertable()) { throw new UnsupportedException ( String.format("INSERT operation on %s tablespace", tableDesc.getUri().toString())); } space.prepareTable(rootNode.getChild()); } }
@Override public void execute(QueryMaster.QueryMasterContext context, QueryContext queryContext, Query query, ExecutionBlockId finalExecBlockId, Path finalOutputDir) throws Exception { Stage lastStage = query.getStage(finalExecBlockId); TableMeta meta = lastStage.getTableMeta(); String nullChar = queryContext.get(SessionVars.NULL_CHAR); meta.putProperty(StorageConstants.TEXT_NULL, nullChar); TableStats stats = lastStage.getResultStats(); TableDesc resultTableDesc = new TableDesc( query.getId().toString(), lastStage.getOutSchema(), meta, finalOutputDir.toUri()); resultTableDesc.setExternal(true); if (query.hasUnionPlan()) { TableStats aggregated = query.aggregateTableStatsOfTerminalBlock(); resultTableDesc.setStats(aggregated); } else { stats.setNumBytes(getTableVolume(query.systemConf, finalOutputDir)); resultTableDesc.setStats(stats); } query.setResultDesc(resultTableDesc); } }
public void init(TableDesc desc, String alias) { this.tableDesc = desc; this.alias = alias; if (!CatalogUtil.isFQTableName(this.tableDesc.getName())) { throw new IllegalArgumentException("the name in TableDesc must be qualified, but it is \"" + desc.getName() + "\""); } String databaseName = CatalogUtil.extractQualifier(this.tableDesc.getName()); String qualifiedAlias = CatalogUtil.buildFQName(databaseName, alias); this.setInSchema(tableDesc.getSchema()); this.getInSchema().setQualifier(qualifiedAlias); this.setOutSchema(new Schema(getInSchema())); logicalSchema = SchemaUtil.getQualifiedLogicalSchema(tableDesc, qualifiedAlias); }
TableDesc table = client.getTableDesc(fqName); if (table.getMeta().getDataFormat().equalsIgnoreCase("SYSTEM")) { continue; if (table.isExternal()) { writer.write(DDLBuilder.buildDDLForExternalTable(table)); } else { if (table.hasPartition()) { writer.write("\n\n"); writer.write("--\n");
if (!desc.isExternal()) { // if the table is the managed table Path oldPath = StorageUtil.concatPath(context.getConf().getVar(TajoConf.ConfVars.WAREHOUSE_DIR), databaseName, simpleTableName); } else { partitionPath = new Path(desc.getUri().toString(), pair.getSecond()); alterTable.setLocation(partitionPath.toString()); Path assumedDirectory = new Path(desc.getUri().toString(), pair.getSecond());
@Override public void purgeTable(TableDesc tableDesc) throws IOException, TajoException { try (HBaseAdmin hAdmin = new HBaseAdmin(hbaseConf)) { HTableDescriptor hTableDesc = parseHTableDescriptor(tableDesc.getMeta(), tableDesc.getSchema()); LOG.info("Deleting hbase table: " + new String(hTableDesc.getName())); hAdmin.disableTable(hTableDesc.getName()); hAdmin.deleteTable(hTableDesc.getName()); } }
public ExternalSortExec(final TaskAttemptContext context,final SortNode plan, final ScanNode scanNode, final CatalogProtos.FragmentProto[] fragments) throws PhysicalPlanningException { this(context, plan); mergedInputFragments = new ArrayList<>(); for (CatalogProtos.FragmentProto proto : fragments) { FileFragment fragment = FragmentConvertor.convert(context.getConf(), proto); mergedInputFragments.add(new Chunk(inSchema, fragment, scanNode.getTableDesc().getMeta())); } }