public static Table convertTable(com.amazonaws.services.glue.model.Table glueTable, String dbName) { requireNonNull(glueTable.getStorageDescriptor(), "Table StorageDescriptor is null"); StorageDescriptor sd = glueTable.getStorageDescriptor(); Table.Builder tableBuilder = Table.builder() .setDatabaseName(dbName) .setTableName(glueTable.getName()) .setOwner(nullToEmpty(glueTable.getOwner())) .setTableType(glueTable.getTableType()) .setDataColumns(sd.getColumns().stream() .map(GlueToPrestoConverter::convertColumn) .collect(toList())) .setParameters(firstNonNull(glueTable.getParameters(), ImmutableMap.of())) .setViewOriginalText(Optional.ofNullable(glueTable.getViewOriginalText())) .setViewExpandedText(Optional.ofNullable(glueTable.getViewExpandedText())); if (glueTable.getPartitionKeys() != null) { tableBuilder.setPartitionColumns(glueTable.getPartitionKeys().stream() .map(GlueToPrestoConverter::convertColumn) .collect(toList())); } else { tableBuilder.setPartitionColumns(new ArrayList<>()); } setStorageBuilder(sd, tableBuilder.getStorageBuilder()); return tableBuilder.build(); }
public static Table fromMetastoreApiTable(org.apache.hadoop.hive.metastore.api.Table table, List<FieldSchema> schema) { StorageDescriptor storageDescriptor = table.getSd(); if (storageDescriptor == null) { throw new PrestoException(HIVE_INVALID_METADATA, "Table is missing storage descriptor"); } Table.Builder tableBuilder = Table.builder() .setDatabaseName(table.getDbName()) .setTableName(table.getTableName()) .setOwner(nullToEmpty(table.getOwner())) .setTableType(table.getTableType()) .setDataColumns(schema.stream() .map(ThriftMetastoreUtil::fromMetastoreApiFieldSchema) .collect(toList())) .setPartitionColumns(table.getPartitionKeys().stream() .map(ThriftMetastoreUtil::fromMetastoreApiFieldSchema) .collect(toList())) .setParameters(table.getParameters() == null ? ImmutableMap.of() : table.getParameters()) .setViewOriginalText(Optional.ofNullable(emptyToNull(table.getViewOriginalText()))) .setViewExpandedText(Optional.ofNullable(emptyToNull(table.getViewExpandedText()))); fromMetastoreApiStorageDescriptor(storageDescriptor, tableBuilder.getStorageBuilder(), table.getTableName()); return tableBuilder.build(); }
.setParameters(tableParameters.build()); tableBuilder.getStorageBuilder() .setStorageFormat(fromHiveStorageFormat(hiveStorageFormat)) .setBucketProperty(bucketProperty)
.setPartitionColumns(partitionColumns); tableBuilder.getStorageBuilder() .setLocation(targetPath.toString()) .setStorageFormat(StorageFormat.create(hiveStorageFormat.getSerDe(), hiveStorageFormat.getInputFormat(), hiveStorageFormat.getOutputFormat()))
@Override public void dropTable(String databaseName, String tableName, boolean deleteData) { try { Optional<Table> table = getTable(databaseName, tableName); if (!table.isPresent()) { throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); } // hack to work around the metastore not being configured for S3 or other FS List<String> locations = listAllDataPaths(databaseName, tableName); Table.Builder tableBuilder = Table.builder(table.get()); tableBuilder.getStorageBuilder().setLocation("/"); // drop table replaceTable(databaseName, tableName, tableBuilder.build(), new PrincipalPrivileges(ImmutableMultimap.of(), ImmutableMultimap.of())); delegate.dropTable(databaseName, tableName, false); // drop data if (deleteData) { for (String location : locations) { Path path = new Path(location); hdfsEnvironment.getFileSystem(TESTING_CONTEXT, path).delete(path, true); } } } catch (IOException e) { throw new UncheckedIOException(e); } finally { invalidateTable(databaseName, tableName); } }
private static Table table( List<Column> partitionColumns, Optional<HiveBucketProperty> bucketProperty) { Table.Builder tableBuilder = Table.builder(); tableBuilder.getStorageBuilder() .setStorageFormat( StorageFormat.create( "com.facebook.hive.orc.OrcSerde", "org.apache.hadoop.hive.ql.io.RCFileInputFormat", "org.apache.hadoop.hive.ql.io.RCFileInputFormat")) .setLocation("hdfs://VOL1:9000/db_name/table_name") .setSkewed(false) .setBucketProperty(bucketProperty); return tableBuilder .setDatabaseName("test_dbname") .setOwner("testOwner") .setTableName("test_table") .setTableType(TableType.MANAGED_TABLE.toString()) .setDataColumns(ImmutableList.of(new Column("col1", HIVE_STRING, Optional.empty()))) .setParameters(ImmutableMap.of()) .setPartitionColumns(partitionColumns) .build(); }
private void alterBucketProperty(SchemaTableName schemaTableName, Optional<HiveBucketProperty> bucketProperty) { try (Transaction transaction = newTransaction()) { ConnectorSession session = newSession(); String tableOwner = session.getUser(); String schemaName = schemaTableName.getSchemaName(); String tableName = schemaTableName.getTableName(); Optional<Table> table = transaction.getMetastore(schemaName).getTable(schemaName, tableName); Table.Builder tableBuilder = Table.builder(table.get()); tableBuilder.getStorageBuilder().setBucketProperty(bucketProperty); PrincipalPrivileges principalPrivileges = testingPrincipalPrivilege(tableOwner); // hack: replaceView can be used as replaceTable despite its name transaction.getMetastore(schemaName).replaceView(schemaName, tableName, tableBuilder.build(), principalPrivileges); transaction.commit(); } }
@Override public void createTable(Table table, PrincipalPrivileges privileges) { // hack to work around the metastore not being configured for S3 or other FS Table.Builder tableBuilder = Table.builder(table); tableBuilder.getStorageBuilder().setLocation("/"); super.createTable(tableBuilder.build(), privileges); }
public void updateTableLocation(String databaseName, String tableName, String location) { Optional<Table> table = getTable(databaseName, tableName); if (!table.isPresent()) { throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); } Table.Builder tableBuilder = Table.builder(table.get()); tableBuilder.getStorageBuilder().setLocation(location); // NOTE: this clears the permissions replaceTable(databaseName, tableName, tableBuilder.build(), new PrincipalPrivileges(ImmutableMultimap.of(), ImmutableMultimap.of())); }
.setViewExpandedText(Optional.of("/* Presto View */")); tableBuilder.getStorageBuilder() .setStorageFormat(VIEW_STORAGE_FORMAT) .setLocation("");
@Override public void dropTable(String databaseName, String tableName, boolean deleteData) { try { Optional<Table> table = getTable(databaseName, tableName); if (!table.isPresent()) { throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); } // hack to work around the metastore not being configured for S3 or other FS List<String> locations = listAllDataPaths(databaseName, tableName); Table.Builder tableBuilder = Table.builder(table.get()); tableBuilder.getStorageBuilder().setLocation("/"); // drop table replaceTable(databaseName, tableName, tableBuilder.build(), new PrincipalPrivileges(ImmutableMultimap.of(), ImmutableMultimap.of())); delegate.dropTable(databaseName, tableName, false); // drop data if (deleteData) { for (String location : locations) { Path path = new Path(location); hdfsEnvironment.getFileSystem(TESTING_CONTEXT, path).delete(path, true); } } } catch (IOException e) { throw new UncheckedIOException(e); } finally { invalidateTable(databaseName, tableName); } }
.setPartitionColumns(partitionColumns); tableBuilder.getStorageBuilder() .setLocation(targetPath.toString()) .setStorageFormat(StorageFormat.create(hiveStorageFormat.getSerDe(), hiveStorageFormat.getInputFormat(), hiveStorageFormat.getOutputFormat()))
private static Table table( List<Column> partitionColumns, Optional<HiveBucketProperty> bucketProperty) { Table.Builder tableBuilder = Table.builder(); tableBuilder.getStorageBuilder() .setStorageFormat( StorageFormat.create( "com.facebook.hive.orc.OrcSerde", "org.apache.hadoop.hive.ql.io.RCFileInputFormat", "org.apache.hadoop.hive.ql.io.RCFileInputFormat")) .setLocation("hdfs://VOL1:9000/db_name/table_name") .setSkewed(false) .setBucketProperty(bucketProperty); return tableBuilder .setDatabaseName("test_dbname") .setOwner("testOwner") .setTableName("test_table") .setTableType(TableType.MANAGED_TABLE.toString()) .setDataColumns(ImmutableList.of(new Column("col1", HIVE_STRING, Optional.empty()))) .setParameters(ImmutableMap.of()) .setPartitionColumns(partitionColumns) .build(); }
private void alterBucketProperty(SchemaTableName schemaTableName, Optional<HiveBucketProperty> bucketProperty) { try (Transaction transaction = newTransaction()) { ConnectorSession session = newSession(); String tableOwner = session.getUser(); String schemaName = schemaTableName.getSchemaName(); String tableName = schemaTableName.getTableName(); Optional<Table> table = transaction.getMetastore(schemaName).getTable(schemaName, tableName); Table.Builder tableBuilder = Table.builder(table.get()); tableBuilder.getStorageBuilder().setBucketProperty(bucketProperty); PrincipalPrivileges principalPrivileges = testingPrincipalPrivilege(tableOwner); // hack: replaceView can be used as replaceTable despite its name transaction.getMetastore(schemaName).replaceView(schemaName, tableName, tableBuilder.build(), principalPrivileges); transaction.commit(); } }
public void updateTableLocation(String databaseName, String tableName, String location) { Optional<Table> table = getTable(databaseName, tableName); if (!table.isPresent()) { throw new TableNotFoundException(new SchemaTableName(databaseName, tableName)); } Table.Builder tableBuilder = Table.builder(table.get()); tableBuilder.getStorageBuilder().setLocation(location); // NOTE: this clears the permissions replaceTable(databaseName, tableName, tableBuilder.build(), new PrincipalPrivileges(ImmutableMultimap.of(), ImmutableMultimap.of())); }
@Override public void createTable(Table table, PrincipalPrivileges privileges) { // hack to work around the metastore not being configured for S3 or other FS Table.Builder tableBuilder = Table.builder(table); tableBuilder.getStorageBuilder().setLocation("/"); super.createTable(tableBuilder.build(), privileges); }