@Override public List<Fragment> getSplits(String inputSourceId, TableDesc table, @Nullable EvalNode filterCondition) throws IOException { return getSplits(inputSourceId, table.getMeta(), table.getSchema(), new Path(table.getUri())); }
@Override public List<Fragment> getSplits(String inputSourceId, TableDesc table, boolean requireSort, @Nullable EvalNode filterCondition) throws IOException { return getSplits(inputSourceId, table.getMeta(), table.getSchema(), requireSort, new Path(table.getUri())); }
private boolean ensureColumnExistance(String tableName, String columnName) throws UndefinedTableException { final TableDesc tableDesc = catalog.getTableDesc(tableName); return tableDesc.getSchema().containsByName(columnName); } }
private boolean ensureColumnExistance(String tableName, String columnName) throws UndefinedTableException { final TableDesc tableDesc = catalog.getTableDesc(tableName); return tableDesc.getSchema().containsByName(columnName); }
public static String buildDDLForBaseTable(TableDesc desc) { StringBuilder sb = new StringBuilder(); sb.append("--\n") .append("-- Name: ").append(CatalogUtil.denormalizeIdentifier(desc.getName())).append("; Type: TABLE;") .append(" Storage: ").append(desc.getMeta().getDataFormat()); sb.append("\n--\n"); sb.append("CREATE TABLE ").append(CatalogUtil.denormalizeIdentifier(desc.getName())); buildSchema(sb, desc.getSchema()); buildUsingClause(sb, desc.getMeta()); buildWithClause(sb, desc.getMeta()); if (desc.hasPartition()) { buildPartitionClause(sb, desc); } sb.append(";"); return sb.toString(); }
public static String buildDDLForBaseTable(TableDesc desc) { StringBuilder sb = new StringBuilder(); sb.append("--\n") .append("-- Name: ").append(IdentifierUtil.denormalizeIdentifier(desc.getName())).append("; Type: TABLE;") .append(" Storage: ").append(desc.getMeta().getDataFormat()); sb.append("\n--\n"); sb.append("CREATE TABLE ").append(IdentifierUtil.denormalizeIdentifier(desc.getName())); buildSchema(sb, desc.getSchema()); buildUsingClause(sb, desc.getMeta()); buildWithClause(sb, desc.getMeta()); if (desc.hasPartition()) { buildPartitionClause(sb, desc); } sb.append(";"); return sb.toString(); }
@Override public void purgeTable(TableDesc tableDesc) throws IOException, TajoException { try (HBaseAdmin hAdmin = new HBaseAdmin(hbaseConf)) { HTableDescriptor hTableDesc = parseHTableDescriptor(tableDesc.getMeta(), tableDesc.getSchema()); LOG.info("Deleting hbase table: " + new String(hTableDesc.getName())); hAdmin.disableTable(hTableDesc.getName()); hAdmin.deleteTable(hTableDesc.getName()); } }
@Override public void purgeTable(TableDesc tableDesc) throws IOException, TajoException { HBaseAdmin hAdmin = new HBaseAdmin(hbaseConf); try { HTableDescriptor hTableDesc = parseHTableDescriptor(tableDesc.getMeta(), tableDesc.getSchema()); LOG.info("Deleting hbase table: " + new String(hTableDesc.getName())); hAdmin.disableTable(hTableDesc.getName()); hAdmin.deleteTable(hTableDesc.getName()); } finally { hAdmin.close(); } }
private static String buildSchemaString(String tableName) throws TajoException { TableDesc desc = client.getTableDesc(tableName); StringBuffer sb = new StringBuffer(); for (Column column : desc.getSchema().getRootColumns()) { sb.append(column.getSimpleName()).append(" ").append(column.getDataType().getType()); TajoDataTypes.DataType dataType = column.getDataType(); if (dataType.getLength() > 0) { sb.append("(").append(dataType.getLength()).append(")"); } sb.append(","); } sb.deleteCharAt(sb.length()-1); return sb.toString(); }
@Override public void createTable(TableDesc tableDesc, boolean ifNotExists) throws TajoException, IOException { createTable(tableDesc.getUri(), tableDesc.getMeta(), tableDesc.getSchema(), tableDesc.isExternal(), ifNotExists); TableStats stats = new TableStats(); stats.setNumRows(TajoConstants.UNKNOWN_ROW_NUMBER); tableDesc.setStats(stats); }
@Override public void createTable(TableDesc tableDesc, boolean ifNotExists) throws TajoException, IOException { createTable(tableDesc.getUri(), tableDesc.getMeta(), tableDesc.getSchema(), tableDesc.isExternal(), ifNotExists); TableStats stats = new TableStats(); stats.setNumRows(TajoConstants.UNKNOWN_ROW_NUMBER); tableDesc.setStats(stats); }
public void assertColumnExists(String tableName,String columnName) throws UndefinedTableException { TableDesc tableDesc = getTableDesc(tableName); assertTrue(tableDesc.getSchema().containsByName(columnName)); }
public void init(TableDesc desc, String alias) { this.tableDesc = desc; this.alias = alias; if (!CatalogUtil.isFQTableName(this.tableDesc.getName())) { throw new IllegalArgumentException("the name in TableDesc must be qualified, but it is \"" + desc.getName() + "\""); } String databaseName = CatalogUtil.extractQualifier(this.tableDesc.getName()); String qualifiedAlias = CatalogUtil.buildFQName(databaseName, alias); this.setInSchema(tableDesc.getSchema()); this.getInSchema().setQualifier(qualifiedAlias); this.setOutSchema(new Schema(getInSchema())); logicalSchema = SchemaUtil.getQualifiedLogicalSchema(tableDesc, qualifiedAlias); }
public void init(TableDesc desc, String alias) { this.tableDesc = desc; this.alias = alias; if (!IdentifierUtil.isFQTableName(this.tableDesc.getName())) { throw new IllegalArgumentException("the name in TableDesc must be qualified, but it is \"" + desc.getName() + "\""); } String databaseName = IdentifierUtil.extractQualifier(this.tableDesc.getName()); String qualifiedAlias = IdentifierUtil.buildFQName(databaseName, alias); this.setInSchema(tableDesc.getSchema()); this.getInSchema().setQualifier(qualifiedAlias); this.setOutSchema(SchemaBuilder.builder().addAll(getInSchema().getRootColumns()).build()); logicalSchema = SchemaUtil.getQualifiedLogicalSchema(tableDesc, qualifiedAlias); }
@Test public final void testInsertOverwriteSmallerColumns() throws Exception { ResultSet res = executeFile("table1_ddl.sql"); res.close(); CatalogService catalog = testingCluster.getMaster().getCatalog(); assertTrue(catalog.existsTable(getCurrentDatabase(), "table1")); TableDesc originalDesc = catalog.getTableDesc(getCurrentDatabase(), "table1"); res = executeFile("testInsertOverwriteSmallerColumns.sql"); res.close(); TableDesc desc = catalog.getTableDesc(getCurrentDatabase(), "table1"); if (!testingCluster.isHiveCatalogStoreRunning()) { assertEquals(8, desc.getStats().getNumRows().intValue()); } assertEquals(originalDesc.getSchema(), desc.getSchema()); executeString("DROP TABLE table1 PURGE"); }
private void createColumnPartitionedTable() throws Exception { String tableName = IdentifierUtil.normalizeIdentifier("ColumnPartitionedTable"); ResultSet res = executeString( "create table " + tableName + " (col1 int4, col2 int4) partition by column(key float8) "); res.close(); assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName)); assertEquals(2, catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName).getSchema().size()); assertEquals(3, catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName).getLogicalSchema().size()); res = testBase.execute( "insert overwrite into " + tableName + " select l_orderkey, l_partkey, l_quantity from lineitem"); res.close(); }
public void setTargetTable(TableDesc desc) { setTableName(desc.getName()); if (desc.hasPartition()) { tableSchema = desc.getLogicalSchema(); } else { tableSchema = desc.getSchema(); } if (desc.getUri() != null) { setUri(desc.getUri()); } setOptions(desc.getMeta().getOptions()); setDataFormat(desc.getMeta().getDataFormat()); if (desc.hasPartition()) { this.setPartitionMethod(desc.getPartitionMethod()); } }
public void setTargetTable(TableDesc desc) { setTableName(desc.getName()); if (desc.hasPartition()) { tableSchema = desc.getLogicalSchema(); } else { tableSchema = desc.getSchema(); } if (desc.getUri() != null) { setUri(desc.getUri()); } setOptions(desc.getMeta().getPropertySet()); setDataFormat(desc.getMeta().getDataFormat()); if (desc.hasPartition()) { this.setPartitionMethod(desc.getPartitionMethod()); } }